From 012dd239ce4d96f67bb92ffc86ea3cfd3cae69fa Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 6 May 2023 16:28:37 +0200 Subject: [PATCH 001/293] docs/api: version-history: also mention /system/df for VirtualSize Commit 1261fe69a3586bb102182aa885197822419c768c deprecated the VirtualSize field, but forgot to mention that it's also included in the /system/df endpoint. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit fdc7a786527b398328dbbbafa3314de11de3fe6b) Signed-off-by: Sebastiaan van Stijn --- docs/api/version-history.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/version-history.md b/docs/api/version-history.md index 530040cdce3bd..f5f20f503d288 100644 --- a/docs/api/version-history.md +++ b/docs/api/version-history.md @@ -23,9 +23,9 @@ keywords: "API, Docker, rcli, REST, documentation" * `GET /images/json` no longer includes hardcoded `:` and `@` in `RepoTags` and`RepoDigests` for untagged images. In such cases, empty arrays will be produced instead. -* The `VirtualSize` field in the `GET /images/{name}/json` and `GET /images//json` - responses is deprecated and will no longer be included in API v1.44. Use the - `Size` field instead, which contains the same information. +* The `VirtualSize` field in the `GET /images/{name}/json`, `GET /images/json`, + and `GET /system/df` responses is deprecated and will no longer be included + in API v1.44. Use the `Size` field instead, which contains the same information. * `GET /info` now includes `no-new-privileges` in the `SecurityOptions` string list when this option is enabled globally. This change is not versioned, and affects all API versions if the daemon has this patch. From 63640838ba6c2d41300cbe87caeb3cbbf66e410e Mon Sep 17 00:00:00 2001 From: Dorin Geman Date: Mon, 8 May 2023 10:22:58 +0300 Subject: [PATCH 002/293] daemon: handleContainerExit(): add execDuration in attributes Add `execDuration` field to the event attributes map. This is useful for tracking how long the container ran. Signed-off-by: Dorin Geman (cherry picked from commit 2ad37e183255e1abf9ca0d06b672338973a04d7b) Signed-off-by: Sebastiaan van Stijn --- daemon/monitor.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/monitor.go b/daemon/monitor.go index c307711e20e21..90998d4643e3c 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -78,7 +78,8 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine } attributes := map[string]string{ - "exitCode": strconv.Itoa(exitStatus.ExitCode), + "exitCode": strconv.Itoa(exitStatus.ExitCode), + "execDuration": strconv.Itoa(int(execDuration.Seconds())), } daemon.Cleanup(c) From d169a5730649e2661c48221f583f8f3c771c7c16 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 8 May 2023 13:17:59 +0200 Subject: [PATCH 003/293] contrib/apparmor: remove remaining version-conditionals (< 2.9) from template Commit 2e19a4d56bf22c99be9d67a1a2f24764aa56e8bb removed all other version- conditional statements from the AppArmor template, but left this one in place. These conditions were added in 8cf89245f5b5f9abb066f599cb69bfe0202bae5d to account for old versions of debian/ubuntu (apparmor_parser < 2.9) that lacked some options; > This allows us to use the apparmor profile we have in contrib/apparmor/ > and solves the problems where certain functions are not apparent on older > versions of apparmor_parser on debian/ubuntu. Those patches were from 2015/2016, and all currently supported distro versions should now have more current versions than that. Looking at the oldest supported versions; Ubuntu 18.04 "Bionic": apparmor_parser --version AppArmor parser version 2.12 Copyright (C) 1999-2008 Novell Inc. Copyright 2009-2012 Canonical Ltd. Debian 10 "Buster" apparmor_parser --version AppArmor parser version 2.13.2 Copyright (C) 1999-2008 Novell Inc. Copyright 2009-2018 Canonical Ltd. This patch removes the remaining conditionals. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit f445ee1e6cba4495e9530b876ec2a213ae595345) Signed-off-by: Sebastiaan van Stijn --- contrib/apparmor/main.go | 16 ++-------------- contrib/apparmor/template.go | 2 -- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/contrib/apparmor/main.go b/contrib/apparmor/main.go index f4a2978b86cb7..d67890d265de7 100644 --- a/contrib/apparmor/main.go +++ b/contrib/apparmor/main.go @@ -6,13 +6,9 @@ import ( "os" "path" "text/template" - - "github.com/docker/docker/pkg/aaparser" ) -type profileData struct { - Version int -} +type profileData struct{} func main() { if len(os.Args) < 2 { @@ -22,15 +18,6 @@ func main() { // parse the arg apparmorProfilePath := os.Args[1] - version, err := aaparser.GetVersion() - if err != nil { - log.Fatal(err) - } - data := profileData{ - Version: version, - } - fmt.Printf("apparmor_parser is of version %+v\n", data) - // parse the template compiled, err := template.New("apparmor_profile").Parse(dockerProfileTemplate) if err != nil { @@ -48,6 +35,7 @@ func main() { } defer f.Close() + data := profileData{} if err := compiled.Execute(f, data); err != nil { log.Fatalf("executing template failed: %v", err) } diff --git a/contrib/apparmor/template.go b/contrib/apparmor/template.go index 4999ca5dc62fc..58afcbe845ee8 100644 --- a/contrib/apparmor/template.go +++ b/contrib/apparmor/template.go @@ -149,9 +149,7 @@ profile /usr/bin/docker (attach_disconnected, complain) { } # xz works via pipes, so we do not need access to the filesystem. profile /usr/bin/xz (complain) { -{{if ge .Version 209000}} signal (receive) peer=/usr/bin/docker, -{{end}} /etc/ld.so.cache r, /lib/** rm, /usr/bin/xz rm, From e28bc0d271a2eeb66a5f768443ae43fc2535c537 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 8 May 2023 13:49:59 +0200 Subject: [PATCH 004/293] profiles/apparmor: remove use of aaparser.GetVersion() commit 7008a514493a83a1342f30afca6726ffea14f50f removed version-conditional rules from the template, so we no longer need the apparmor_parser Version. This patch removes the call to `aaparser.GetVersion()` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ecaab085db4b4a7a0ba5bbf224ec483aefa9ee09) Signed-off-by: Sebastiaan van Stijn --- profiles/apparmor/apparmor.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/profiles/apparmor/apparmor.go b/profiles/apparmor/apparmor.go index b3566b2f7354d..d0f2361605065 100644 --- a/profiles/apparmor/apparmor.go +++ b/profiles/apparmor/apparmor.go @@ -14,10 +14,8 @@ import ( "github.com/docker/docker/pkg/aaparser" ) -var ( - // profileDirectory is the file store for apparmor profiles and macros. - profileDirectory = "/etc/apparmor.d" -) +// profileDirectory is the file store for apparmor profiles and macros. +const profileDirectory = "/etc/apparmor.d" // profileData holds information about the given profile for generation. type profileData struct { @@ -29,8 +27,6 @@ type profileData struct { Imports []string // InnerImports defines the apparmor functions to import in the profile. InnerImports []string - // Version is the {major, minor, patch} version of apparmor_parser as a single number. - Version int } // generateDefault creates an apparmor profile from ProfileData. @@ -50,12 +46,6 @@ func (p *profileData) generateDefault(out io.Writer) error { p.InnerImports = append(p.InnerImports, "#include ") } - ver, err := aaparser.GetVersion() - if err != nil { - return err - } - p.Version = ver - return compiled.Execute(out, p) } From bfffb0974e92928764845df935d092e6bdcb542d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 8 May 2023 13:53:30 +0200 Subject: [PATCH 005/293] pkg/aaparser: deprecate GetVersion, as it's no longer used Our templates no longer contain version-specific rules, so this function is no longer used. This patch deprecates it. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit e3e715666f95c056390a88e0f3d1033a1aac2762) Signed-off-by: Sebastiaan van Stijn --- pkg/aaparser/aaparser.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/aaparser/aaparser.go b/pkg/aaparser/aaparser.go index 2b5a2605f9c12..3d7c2c5a97b3c 100644 --- a/pkg/aaparser/aaparser.go +++ b/pkg/aaparser/aaparser.go @@ -13,6 +13,8 @@ const ( ) // GetVersion returns the major and minor version of apparmor_parser. +// +// Deprecated: no longer used, and will be removed in the next release. func GetVersion() (int, error) { output, err := cmd("", "--version") if err != nil { From 86770904be72ba755f5d80f93eede77a1a5a74df Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Tue, 9 May 2023 16:19:05 +0100 Subject: [PATCH 006/293] c8d: fix missing image history Signed-off-by: Laura Brehm (cherry picked from commit e8be7921302241b885395322d4a47a57fb2f21cd) Signed-off-by: Laura Brehm --- daemon/containerd/image.go | 16 ++++++++++++++++ daemon/containerd/image_commit.go | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index d15369185e2ca..8ced6436f5be3 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -68,6 +68,21 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima exposedPorts[nat.Port(k)] = v } + var imgHistory []image.History + for _, h := range ociimage.History { + var created time.Time + if h.Created != nil { + created = *h.Created + } + imgHistory = append(imgHistory, image.History{ + Created: created, + Author: h.Author, + CreatedBy: h.CreatedBy, + Comment: h.Comment, + EmptyLayer: h.EmptyLayer, + }) + } + img := image.NewImage(image.ID(desc.Digest)) img.V1Image = image.V1Image{ ID: string(desc.Digest), @@ -87,6 +102,7 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima } img.RootFS = rootfs + img.History = imgHistory if options.Details { lastUpdated := time.Unix(0, 0) diff --git a/daemon/containerd/image_commit.go b/daemon/containerd/image_commit.go index e9dc49d921727..bbb4df0ddc467 100644 --- a/daemon/containerd/image_commit.go +++ b/daemon/containerd/image_commit.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "runtime" + "strings" "time" "github.com/containerd/containerd/content" @@ -143,7 +144,7 @@ func generateCommitImageConfig(baseConfig ocispec.Image, diffID digest.Digest, o }, History: append(baseConfig.History, ocispec.History{ Created: &createdTime, - CreatedBy: "", // FIXME(ndeloof) ? + CreatedBy: strings.Join(opts.ContainerConfig.Cmd, " "), Author: opts.Author, Comment: opts.Comment, EmptyLayer: diffID == "", From 763d2b7996021f7dae14196649e27ddc225eaa9c Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Wed, 10 May 2023 01:33:15 +0100 Subject: [PATCH 007/293] c8d: fix image history for dangling images Signed-off-by: Laura Brehm (cherry picked from commit 4603b6d6b6b00bb635a03ffcb693cde32b3f5c7d) Signed-off-by: Laura Brehm --- daemon/containerd/image_history.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/daemon/containerd/image_history.go b/daemon/containerd/image_history.go index a99716ead73be..a2d0c11425b4c 100644 --- a/daemon/containerd/image_history.go +++ b/daemon/containerd/image_history.go @@ -90,13 +90,16 @@ func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imaget return nil, err } - tags := make([]string, len(tagged)) - for i, t := range tagged { + var tags []string + for _, t := range tagged { + if isDanglingImage(t) { + continue + } name, err := reference.ParseNamed(t.Name) if err != nil { return nil, err } - tags[i] = reference.FamiliarString(name) + tags = append(tags, reference.FamiliarString(name)) } history[0].Tags = tags } From 1235338836207672ca7ab837b4eb6b5794ed64e5 Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Tue, 25 Apr 2023 22:43:51 +0100 Subject: [PATCH 008/293] c8d: implement missing image delete logic Ports over all the previous image delete logic, such as: - Introduce `prune` and `force` flags - Introduce the concept of hard and soft image delete conflics, which represent: - image referenced in multiple tags (soft conflict) - image being used by a stopped container (soft conflict) - image being used by a running container (hard conflict) - Implement delete logic such as: - if deleting by reference, and there are other references to the same image, just delete the passed reference - if deleting by reference, and there is only 1 reference and the image is being used by a running container, throw an error if !force, or delete the reference and create a dangling reference otherwise - if deleting by imageID, and force is true, remove all tags (otherwise soft conflict) - if imageID, check if stopped container is using the image (soft conflict), and delete anyway if force - if imageID was passed in, check if running container is using the image (hard conflict) - if `prune` is true, and the image being deleted has dangling parents, remove them This commit also implements logic to get image parents in c8d by comparing shared layers. Signed-off-by: Laura Brehm (cherry picked from commit cad97135b32bbede9e38d0ac4d8f16b49fa4b3ee) Signed-off-by: Laura Brehm --- daemon/containerd/image_children.go | 68 +++++++ daemon/containerd/image_delete.go | 280 ++++++++++++++++++++++++++-- 2 files changed, 334 insertions(+), 14 deletions(-) diff --git a/daemon/containerd/image_children.go b/daemon/containerd/image_children.go index ced4637aebbd5..ba8c93954917e 100644 --- a/daemon/containerd/image_children.go +++ b/daemon/containerd/image_children.go @@ -127,3 +127,71 @@ func isRootfsChildOf(child ocispec.RootFS, parent ocispec.RootFS) bool { return true } + +// parents returns a slice of image IDs whose entire rootfs contents match, +// in order, the childs first layers, excluding images with the exact same +// rootfs. +// +// Called from image_delete.go to prune dangling parents. +func (i *ImageService) parents(ctx context.Context, id image.ID) ([]imageWithRootfs, error) { + target, err := i.resolveDescriptor(ctx, id.String()) + if err != nil { + return nil, errors.Wrap(err, "failed to get child image") + } + + cs := i.client.ContentStore() + + allPlatforms, err := containerdimages.Platforms(ctx, cs, target) + if err != nil { + return nil, errdefs.System(errors.Wrap(err, "failed to list platforms supported by image")) + } + + var childRootFS []ocispec.RootFS + for _, platform := range allPlatforms { + rootfs, err := platformRootfs(ctx, cs, target, platform) + if err != nil { + if cerrdefs.IsNotFound(err) { + continue + } + return nil, errdefs.System(errors.Wrap(err, "failed to get platform-specific rootfs")) + } + + childRootFS = append(childRootFS, rootfs) + } + + imgs, err := i.client.ImageService().List(ctx) + if err != nil { + return nil, errdefs.System(errors.Wrap(err, "failed to list all images")) + } + + var parents []imageWithRootfs + for _, img := range imgs { + nextImage: + for _, platform := range allPlatforms { + rootfs, err := platformRootfs(ctx, cs, img.Target, platform) + if err != nil { + if cerrdefs.IsNotFound(err) { + continue + } + return nil, errdefs.System(errors.Wrap(err, "failed to get platform-specific rootfs")) + } + + for _, childRoot := range childRootFS { + if isRootfsChildOf(childRoot, rootfs) { + parents = append(parents, imageWithRootfs{ + img: img, + rootfs: rootfs, + }) + break nextImage + } + } + } + } + + return parents, nil +} + +type imageWithRootfs struct { + img containerdimages.Image + rootfs ocispec.RootFS +} diff --git a/daemon/containerd/image_delete.go b/daemon/containerd/image_delete.go index 1fd13cf4201ca..3d55dba3948ca 100644 --- a/daemon/containerd/image_delete.go +++ b/daemon/containerd/image_delete.go @@ -2,10 +2,16 @@ package containerd import ( "context" + "fmt" + "sort" + "strings" "github.com/containerd/containerd/images" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" + "github.com/docker/docker/container" + "github.com/docker/docker/image" + "github.com/docker/docker/pkg/stringid" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" @@ -30,8 +36,6 @@ import ( // are divided into two categories grouped by their severity: // // Hard Conflict: -// - a pull or build using the image. -// - any descendant image. // - any running container using the image. // // Soft Conflict: @@ -45,8 +49,6 @@ import ( // meaning any delete conflicts will cause the image to not be deleted and the // conflict will not be reported. // -// TODO(thaJeztah): implement ImageDelete "force" options; see https://github.com/moby/moby/issues/43850 -// TODO(thaJeztah): implement ImageDelete "prune" options; see https://github.com/moby/moby/issues/43849 // TODO(thaJeztah): image delete should send prometheus counters; see https://github.com/moby/moby/issues/45268 func (i *ImageService) ImageDelete(ctx context.Context, imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { parsedRef, err := reference.ParseNormalizedNamed(imageRef) @@ -59,28 +61,278 @@ func (i *ImageService) ImageDelete(ctx context.Context, imageRef string, force, return nil, err } + imgID := image.ID(img.Target.Digest) + + if isImageIDPrefix(imgID.String(), imageRef) { + return i.deleteAll(ctx, img, force, prune) + } + + singleRef, err := i.isSingleReference(ctx, img) + if err != nil { + return nil, err + } + if !singleRef { + err := i.client.ImageService().Delete(ctx, img.Name) + if err != nil { + return nil, err + } + i.LogImageEvent(imgID.String(), imgID.String(), "untag") + records := []types.ImageDeleteResponseItem{{Untagged: reference.FamiliarString(reference.TagNameOnly(parsedRef))}} + return records, nil + } + + using := func(c *container.Container) bool { + return c.ImageID == imgID + } + ctr := i.containers.First(using) + if ctr != nil { + if !force { + // If we removed the repository reference then + // this image would remain "dangling" and since + // we really want to avoid that the client must + // explicitly force its removal. + refString := reference.FamiliarString(reference.TagNameOnly(parsedRef)) + err := &imageDeleteConflict{ + reference: refString, + used: true, + message: fmt.Sprintf("container %s is using its referenced image %s", + stringid.TruncateID(ctr.ID), + stringid.TruncateID(imgID.String())), + } + return nil, err + } + + err := i.softImageDelete(ctx, img) + if err != nil { + return nil, err + } + + i.LogImageEvent(imgID.String(), imgID.String(), "untag") + records := []types.ImageDeleteResponseItem{{Untagged: reference.FamiliarString(reference.TagNameOnly(parsedRef))}} + return records, nil + } + + return i.deleteAll(ctx, img, force, prune) +} + +// deleteAll deletes the image from the daemon, and if prune is true, +// also deletes dangling parents if there is no conflict in doing so. +// Parent images are removed quietly, and if there is any issue/conflict +// it is logged but does not halt execution/an error is not returned. +func (i *ImageService) deleteAll(ctx context.Context, img images.Image, force, prune bool) ([]types.ImageDeleteResponseItem, error) { + var records []types.ImageDeleteResponseItem + + // Workaround for: https://github.com/moby/buildkit/issues/3797 possiblyDeletedConfigs := map[digest.Digest]struct{}{} - if err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) { + err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) { if images.IsConfigType(d.MediaType) { possiblyDeletedConfigs[d.Digest] = struct{}{} } - }); err != nil { + }) + if err != nil { return nil, err } + defer func() { + if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil { + logrus.WithError(err).Warn("failed to unlease snapshots") + } + }() - err = i.client.ImageService().Delete(ctx, img.Name, images.SynchronousDelete()) + imgID := img.Target.Digest.String() + + var parents []imageWithRootfs + if prune { + parents, err = i.parents(ctx, image.ID(imgID)) + if err != nil { + logrus.WithError(err).Warn("failed to get image parents") + } + sortParentsByAffinity(parents) + } + + imageRefs, err := i.client.ImageService().List(ctx, "target.digest=="+imgID) if err != nil { return nil, err } + for _, imageRef := range imageRefs { + if err := i.imageDeleteHelper(ctx, imageRef, &records, force); err != nil { + return records, err + } + } + i.LogImageEvent(imgID, imgID, "delete") + records = append(records, types.ImageDeleteResponseItem{Deleted: imgID}) - // Workaround for: https://github.com/moby/buildkit/issues/3797 - if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil { - logrus.WithError(err).Warn("failed to unlease snapshots") + for _, parent := range parents { + if !isDanglingImage(parent.img) { + break + } + err = i.imageDeleteHelper(ctx, parent.img, &records, false) + if err != nil { + logrus.WithError(err).Warn("failed to remove image parent") + break + } + parentID := parent.img.Target.Digest.String() + i.LogImageEvent(parentID, parentID, "delete") + records = append(records, types.ImageDeleteResponseItem{Deleted: parentID}) } - imgID := string(img.Target.Digest) - i.LogImageEvent(imgID, imgID, "untag") - i.LogImageEvent(imgID, imgID, "delete") + return records, nil +} + +// isImageIDPrefix returns whether the given +// possiblePrefix is a prefix of the given imageID. +func isImageIDPrefix(imageID, possiblePrefix string) bool { + if strings.HasPrefix(imageID, possiblePrefix) { + return true + } + if i := strings.IndexRune(imageID, ':'); i >= 0 { + return strings.HasPrefix(imageID[i+1:], possiblePrefix) + } + return false +} + +func sortParentsByAffinity(parents []imageWithRootfs) { + sort.Slice(parents, func(i, j int) bool { + lenRootfsI := len(parents[i].rootfs.DiffIDs) + lenRootfsJ := len(parents[j].rootfs.DiffIDs) + if lenRootfsI == lenRootfsJ { + return isDanglingImage(parents[i].img) + } + return lenRootfsI > lenRootfsJ + }) +} + +// isSingleReference returns true if there are no other images in the +// daemon targeting the same content as `img` that are not dangling. +func (i *ImageService) isSingleReference(ctx context.Context, img images.Image) (bool, error) { + refs, err := i.client.ImageService().List(ctx, "target.digest=="+img.Target.Digest.String()) + if err != nil { + return false, err + } + for _, ref := range refs { + if !isDanglingImage(ref) && ref.Name != img.Name { + return false, nil + } + } + return true, nil +} + +type conflictType int + +const ( + conflictRunningContainer conflictType = 1 << iota + conflictActiveReference + conflictStoppedContainer + conflictHard = conflictRunningContainer + conflictSoft = conflictActiveReference | conflictStoppedContainer +) + +// imageDeleteHelper attempts to delete the given image from this daemon. +// If the image has any hard delete conflicts (running containers using +// the image) then it cannot be deleted. If the image has any soft delete +// conflicts (any tags/digests referencing the image or any stopped container +// using the image) then it can only be deleted if force is true. Any deleted +// images and untagged references are appended to the given records. If any +// error or conflict is encountered, it will be returned immediately without +// deleting the image. +func (i *ImageService) imageDeleteHelper(ctx context.Context, img images.Image, records *[]types.ImageDeleteResponseItem, force bool) error { + // First, determine if this image has any conflicts. Ignore soft conflicts + // if force is true. + c := conflictHard + if !force { + c |= conflictSoft + } + + imgID := image.ID(img.Target.Digest) + + err := i.checkImageDeleteConflict(ctx, imgID, c) + if err != nil { + return err + } + + untaggedRef, err := reference.ParseAnyReference(img.Name) + if err != nil { + return err + } + err = i.client.ImageService().Delete(ctx, img.Name, images.SynchronousDelete()) + if err != nil { + return err + } + + i.LogImageEvent(imgID.String(), imgID.String(), "untag") + *records = append(*records, types.ImageDeleteResponseItem{Untagged: reference.FamiliarString(untaggedRef)}) + + return nil +} + +// ImageDeleteConflict holds a soft or hard conflict and associated +// error. A hard conflict represents a running container using the +// image, while a soft conflict is any tags/digests referencing the +// given image or any stopped container using the image. +// Implements the error interface. +type imageDeleteConflict struct { + hard bool + used bool + reference string + message string +} + +func (idc *imageDeleteConflict) Error() string { + var forceMsg string + if idc.hard { + forceMsg = "cannot be forced" + } else { + forceMsg = "must be forced" + } + return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", idc.reference, forceMsg, idc.message) +} + +func (imageDeleteConflict) Conflict() {} + +// checkImageDeleteConflict returns a conflict representing +// any issue preventing deletion of the given image ID, and +// nil if there are none. It takes a bitmask representing a +// filter for which conflict types the caller cares about, +// and will only check for these conflict types. +func (i *ImageService) checkImageDeleteConflict(ctx context.Context, imgID image.ID, mask conflictType) error { + if mask&conflictRunningContainer != 0 { + running := func(c *container.Container) bool { + return c.ImageID == imgID && c.IsRunning() + } + if ctr := i.containers.First(running); ctr != nil { + return &imageDeleteConflict{ + reference: stringid.TruncateID(imgID.String()), + hard: true, + used: true, + message: fmt.Sprintf("image is being used by running container %s", stringid.TruncateID(ctr.ID)), + } + } + } + + if mask&conflictStoppedContainer != 0 { + stopped := func(c *container.Container) bool { + return !c.IsRunning() && c.ImageID == imgID + } + if ctr := i.containers.First(stopped); ctr != nil { + return &imageDeleteConflict{ + reference: stringid.TruncateID(imgID.String()), + used: true, + message: fmt.Sprintf("image is being used by stopped container %s", stringid.TruncateID(ctr.ID)), + } + } + } + + if mask&conflictActiveReference != 0 { + refs, err := i.client.ImageService().List(ctx, "target.digest=="+imgID.String()) + if err != nil { + return err + } + if len(refs) > 1 { + return &imageDeleteConflict{ + reference: stringid.TruncateID(imgID.String()), + message: "image is referenced in multiple repositories", + } + } + } - return []types.ImageDeleteResponseItem{{Untagged: reference.FamiliarString(parsedRef)}}, nil + return nil } From ecbc27aa223243cd0ad415b50cdf6ecb72f2395d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 11 May 2023 00:38:04 +0200 Subject: [PATCH 009/293] vendor: github.com/docker/distribution v2.8.2 CI - Dockerfile: fix filenames of artifacts Bugfixes - Fix panic in inmemory driver - Add code to handle pagination of parts. Fixes max layer size of 10GB bug - Parse http forbidden as denied - Revert "registry/client: set Accept: identity header when getting layers Runtime - Update to go1.19.9 - Dockerfile: update xx to v1.2.1 ([#3907](https://github.com/distribution/distribution/pull/3907)) Security - Fix [CVE-2022-28391](https://www.cve.org/CVERecord?id=CVE-2022-28391) by bumping alpine from 3.14 to 3.16 - Fix [CVE-2023-2253](https://www.cve.org/CVERecord?id=CVE-2023-2253) runaway allocation on /v2/_catalog [`521ea3d9`](https://github.com/distribution/distribution/commit/521ea3d973cb0c7089ebbcdd4ccadc34be941f54) full diff: https://github.com/docker/distribution/compare/v2.8.1...v2.8.2 Signed-off-by: Sebastiaan van Stijn bump to release/2.8 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 7821d2d78878e034e8998faadde3f10de5eb801b) Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 +- .../docker/distribution/.dockerignore | 1 + .../docker/distribution/.golangci.yml | 7 ++ .../github.com/docker/distribution/.mailmap | 6 +- .../github.com/docker/distribution/Dockerfile | 78 +++++++++++-------- .../github.com/docker/distribution/Makefile | 2 +- .../docker/distribution/docker-bake.hcl | 21 ++--- .../distribution/reference/reference.go | 4 +- .../registry/api/v2/descriptors.go | 17 ++++ .../distribution/registry/api/v2/errors.go | 9 +++ .../distribution/registry/client/errors.go | 2 + .../registry/client/repository.go | 4 +- .../registry/client/transport/http_reader.go | 1 - vendor/modules.txt | 2 +- 15 files changed, 98 insertions(+), 62 deletions(-) create mode 100644 vendor/github.com/docker/distribution/.dockerignore diff --git a/vendor.mod b/vendor.mod index 17d7485cfb2e3..44d53f8c91cac 100644 --- a/vendor.mod +++ b/vendor.mod @@ -32,7 +32,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/creack/pty v1.1.18 github.com/deckarep/golang-set/v2 v2.3.0 - github.com/docker/distribution v2.8.1+incompatible + github.com/docker/distribution v2.8.2+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c github.com/docker/go-metrics v0.0.1 diff --git a/vendor.sum b/vendor.sum index ac001bd255460..2d78b1393f75b 100644 --- a/vendor.sum +++ b/vendor.sum @@ -502,8 +502,8 @@ github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TT github.com/docker/distribution v2.6.0-rc.1.0.20180327202408-83389a148052+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.0.0-20200511152416-a93e9eb0e95c/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20180531152204-71cd53e4a197/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= diff --git a/vendor/github.com/docker/distribution/.dockerignore b/vendor/github.com/docker/distribution/.dockerignore new file mode 100644 index 0000000000000..e660fd93d3196 --- /dev/null +++ b/vendor/github.com/docker/distribution/.dockerignore @@ -0,0 +1 @@ +bin/ diff --git a/vendor/github.com/docker/distribution/.golangci.yml b/vendor/github.com/docker/distribution/.golangci.yml index 1ba6cb91623e0..36c083b0fc43a 100644 --- a/vendor/github.com/docker/distribution/.golangci.yml +++ b/vendor/github.com/docker/distribution/.golangci.yml @@ -18,3 +18,10 @@ run: deadline: 2m skip-dirs: - vendor + +issues: + exclude-rules: + # io/ioutil is deprecated, but won't be removed until Go v2. It's safe to ignore for the release/2.8 branch. + - text: "SA1019: \"io/ioutil\" has been deprecated since Go 1.16" + linters: + - staticcheck diff --git a/vendor/github.com/docker/distribution/.mailmap b/vendor/github.com/docker/distribution/.mailmap index 8f3738f3d0e1d..d94c3936e0219 100644 --- a/vendor/github.com/docker/distribution/.mailmap +++ b/vendor/github.com/docker/distribution/.mailmap @@ -44,6 +44,8 @@ Thomas Berger Thomas Berger Samuel Karp Samuel Karp Justin Cormack sayboras -CrazyMax CrazyMax <1951866+crazy-max@users.noreply.github.com> -CrazyMax +Hayley Swimelar +Jose D. Gomez R +Shengjing Zhu +Silvin Lubecki <31478878+silvin-lubecki@users.noreply.github.com> diff --git a/vendor/github.com/docker/distribution/Dockerfile b/vendor/github.com/docker/distribution/Dockerfile index ae8c040c735c7..fb54b68138d04 100644 --- a/vendor/github.com/docker/distribution/Dockerfile +++ b/vendor/github.com/docker/distribution/Dockerfile @@ -1,49 +1,59 @@ -# syntax=docker/dockerfile:1.3 +# syntax=docker/dockerfile:1 -ARG GO_VERSION=1.16.15 -ARG GORELEASER_XX_VERSION=1.2.5 +ARG GO_VERSION=1.19.9 +ARG ALPINE_VERSION=3.16 +ARG XX_VERSION=1.2.1 -FROM --platform=$BUILDPLATFORM crazymax/goreleaser-xx:${GORELEASER_XX_VERSION} AS goreleaser-xx -FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS base -COPY --from=goreleaser-xx / / -RUN apk add --no-cache file git +FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS base +COPY --from=xx / / +RUN apk add --no-cache bash coreutils file git +ENV GO111MODULE=auto +ENV CGO_ENABLED=0 WORKDIR /go/src/github.com/docker/distribution +FROM base AS version +ARG PKG="github.com/docker/distribution" +RUN --mount=target=. \ + VERSION=$(git describe --match 'v[0-9]*' --dirty='.m' --always --tags) REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi); \ + echo "-X ${PKG}/version.Version=${VERSION#v} -X ${PKG}/version.Revision=${REVISION} -X ${PKG}/version.Package=${PKG}" | tee /tmp/.ldflags; \ + echo -n "${VERSION}" | tee /tmp/.version; + FROM base AS build -ENV GO111MODULE=auto -ENV CGO_ENABLED=0 -# GIT_REF is used by goreleaser-xx to handle the proper git ref when available. -# It will fallback to the working tree info if empty and use "git tag --points-at" -# or "git describe" to define the version info. -ARG GIT_REF ARG TARGETPLATFORM -ARG PKG="github.com/distribution/distribution" +ARG LDFLAGS="-s -w" ARG BUILDTAGS="include_oss include_gcs" -RUN --mount=type=bind,rw \ - --mount=type=cache,target=/root/.cache/go-build \ - --mount=target=/go/pkg/mod,type=cache \ - goreleaser-xx --debug \ - --name="registry" \ - --dist="/out" \ - --main="./cmd/registry" \ - --flags="-v" \ - --ldflags="-s -w -X '$PKG/version.Version={{.Version}}' -X '$PKG/version.Revision={{.Commit}}' -X '$PKG/version.Package=$PKG'" \ - --tags="$BUILDTAGS" \ - --files="LICENSE" \ - --files="README.md" - -FROM scratch AS artifact -COPY --from=build /out/*.tar.gz / -COPY --from=build /out/*.zip / -COPY --from=build /out/*.sha256 / +RUN --mount=type=bind,target=/go/src/github.com/docker/distribution,rw \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=target=/go/pkg/mod,type=cache \ + --mount=type=bind,source=/tmp/.ldflags,target=/tmp/.ldflags,from=version \ + set -x ; xx-go build -trimpath -ldflags "$(cat /tmp/.ldflags) ${LDFLAGS}" -o /usr/bin/registry ./cmd/registry \ + && xx-verify --static /usr/bin/registry FROM scratch AS binary -COPY --from=build /usr/local/bin/registry* / +COPY --from=build /usr/bin/registry / + +FROM base AS releaser +ARG TARGETOS +ARG TARGETARCH +ARG TARGETVARIANT +WORKDIR /work +RUN --mount=from=binary,target=/build \ + --mount=type=bind,target=/src \ + --mount=type=bind,source=/tmp/.version,target=/tmp/.version,from=version \ + VERSION=$(cat /tmp/.version) \ + && mkdir -p /out \ + && cp /build/registry /src/README.md /src/LICENSE . \ + && tar -czvf "/out/registry_${VERSION#v}_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}.tar.gz" * \ + && sha256sum -z "/out/registry_${VERSION#v}_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}.tar.gz" | awk '{ print $1 }' > "/out/registry_${VERSION#v}_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}.tar.gz.sha256" + +FROM scratch AS artifact +COPY --from=releaser /out / -FROM alpine:3.14 +FROM alpine:${ALPINE_VERSION} RUN apk add --no-cache ca-certificates COPY cmd/registry/config-dev.yml /etc/docker/registry/config.yml -COPY --from=build /usr/local/bin/registry /bin/registry +COPY --from=binary /registry /bin/registry VOLUME ["/var/lib/registry"] EXPOSE 5000 ENTRYPOINT ["registry"] diff --git a/vendor/github.com/docker/distribution/Makefile b/vendor/github.com/docker/distribution/Makefile index 331da27328d18..75e11820152cc 100644 --- a/vendor/github.com/docker/distribution/Makefile +++ b/vendor/github.com/docker/distribution/Makefile @@ -50,7 +50,7 @@ version/version.go: check: ## run all linters (TODO: enable "unused", "varcheck", "ineffassign", "unconvert", "staticheck", "goimports", "structcheck") @echo "$(WHALE) $@" - golangci-lint run + @GO111MODULE=off golangci-lint run test: ## run tests, except integration test with test.short @echo "$(WHALE) $@" diff --git a/vendor/github.com/docker/distribution/docker-bake.hcl b/vendor/github.com/docker/distribution/docker-bake.hcl index 4dd5a100c1de3..91686e608a9c5 100644 --- a/vendor/github.com/docker/distribution/docker-bake.hcl +++ b/vendor/github.com/docker/distribution/docker-bake.hcl @@ -1,15 +1,3 @@ -// GITHUB_REF is the actual ref that triggers the workflow -// https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables -variable "GITHUB_REF" { - default = "" -} - -target "_common" { - args = { - GIT_REF = GITHUB_REF - } -} - group "default" { targets = ["image-local"] } @@ -20,13 +8,11 @@ target "docker-metadata-action" { } target "binary" { - inherits = ["_common"] target = "binary" output = ["./bin"] } target "artifact" { - inherits = ["_common"] target = "artifact" output = ["./bin"] } @@ -43,8 +29,13 @@ target "artifact-all" { ] } +// Special target: https://github.com/docker/metadata-action#bake-definition +target "docker-metadata-action" { + tags = ["registry:local"] +} + target "image" { - inherits = ["_common", "docker-metadata-action"] + inherits = ["docker-metadata-action"] } target "image-local" { diff --git a/vendor/github.com/docker/distribution/reference/reference.go b/vendor/github.com/docker/distribution/reference/reference.go index 8c0c23b2fe1b7..b7cd00b0d68e2 100644 --- a/vendor/github.com/docker/distribution/reference/reference.go +++ b/vendor/github.com/docker/distribution/reference/reference.go @@ -3,13 +3,13 @@ // // Grammar // -// reference := name [ ":" tag ] [ "@" digest ] +// reference := name [ ":" tag ] [ "@" digest ] // name := [domain '/'] path-component ['/' path-component]* // domain := domain-component ['.' domain-component]* [':' port-number] // domain-component := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/ // port-number := /[0-9]+/ // path-component := alpha-numeric [separator alpha-numeric]* -// alpha-numeric := /[a-z0-9]+/ +// alpha-numeric := /[a-z0-9]+/ // separator := /[_.]|__|[-]*/ // // tag := /[\w][\w.-]{0,127}/ diff --git a/vendor/github.com/docker/distribution/registry/api/v2/descriptors.go b/vendor/github.com/docker/distribution/registry/api/v2/descriptors.go index a9616c58ad59f..c3bf90f71d0a2 100644 --- a/vendor/github.com/docker/distribution/registry/api/v2/descriptors.go +++ b/vendor/github.com/docker/distribution/registry/api/v2/descriptors.go @@ -134,6 +134,19 @@ var ( }, } + invalidPaginationResponseDescriptor = ResponseDescriptor{ + Name: "Invalid pagination number", + Description: "The received parameter n was invalid in some way, as described by the error code. The client should resolve the issue and retry the request.", + StatusCode: http.StatusBadRequest, + Body: BodyDescriptor{ + ContentType: "application/json", + Format: errorsBody, + }, + ErrorCodes: []errcode.ErrorCode{ + ErrorCodePaginationNumberInvalid, + }, + } + repositoryNotFoundResponseDescriptor = ResponseDescriptor{ Name: "No Such Repository Error", StatusCode: http.StatusNotFound, @@ -490,6 +503,7 @@ var routeDescriptors = []RouteDescriptor{ }, }, Failures: []ResponseDescriptor{ + invalidPaginationResponseDescriptor, unauthorizedResponseDescriptor, repositoryNotFoundResponseDescriptor, deniedResponseDescriptor, @@ -1578,6 +1592,9 @@ var routeDescriptors = []RouteDescriptor{ }, }, }, + Failures: []ResponseDescriptor{ + invalidPaginationResponseDescriptor, + }, }, }, }, diff --git a/vendor/github.com/docker/distribution/registry/api/v2/errors.go b/vendor/github.com/docker/distribution/registry/api/v2/errors.go index 97d6923aa0321..87e9f3c14be06 100644 --- a/vendor/github.com/docker/distribution/registry/api/v2/errors.go +++ b/vendor/github.com/docker/distribution/registry/api/v2/errors.go @@ -133,4 +133,13 @@ var ( longer proceed.`, HTTPStatusCode: http.StatusNotFound, }) + + ErrorCodePaginationNumberInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "PAGINATION_NUMBER_INVALID", + Message: "invalid number of results requested", + Description: `Returned when the "n" parameter (number of results + to return) is not an integer, "n" is negative or "n" is bigger than + the maximum allowed.`, + HTTPStatusCode: http.StatusBadRequest, + }) ) diff --git a/vendor/github.com/docker/distribution/registry/client/errors.go b/vendor/github.com/docker/distribution/registry/client/errors.go index 52d49d5d295f2..024df43dd9204 100644 --- a/vendor/github.com/docker/distribution/registry/client/errors.go +++ b/vendor/github.com/docker/distribution/registry/client/errors.go @@ -55,6 +55,8 @@ func parseHTTPErrorResponse(statusCode int, r io.Reader) error { switch statusCode { case http.StatusUnauthorized: return errcode.ErrorCodeUnauthorized.WithMessage(detailsErr.Details) + case http.StatusForbidden: + return errcode.ErrorCodeDenied.WithMessage(detailsErr.Details) case http.StatusTooManyRequests: return errcode.ErrorCodeTooManyRequests.WithMessage(detailsErr.Details) default: diff --git a/vendor/github.com/docker/distribution/registry/client/repository.go b/vendor/github.com/docker/distribution/registry/client/repository.go index 3e2ae66d3cf8c..04e5a3ba01f32 100644 --- a/vendor/github.com/docker/distribution/registry/client/repository.go +++ b/vendor/github.com/docker/distribution/registry/client/repository.go @@ -114,9 +114,7 @@ func (r *registry) Repositories(ctx context.Context, entries []string, last stri return 0, err } - for cnt := range ctlg.Repositories { - entries[cnt] = ctlg.Repositories[cnt] - } + copy(entries, ctlg.Repositories) numFilled = len(ctlg.Repositories) link := resp.Header.Get("Link") diff --git a/vendor/github.com/docker/distribution/registry/client/transport/http_reader.go b/vendor/github.com/docker/distribution/registry/client/transport/http_reader.go index 1d0b382fb511b..9120dbed666e4 100644 --- a/vendor/github.com/docker/distribution/registry/client/transport/http_reader.go +++ b/vendor/github.com/docker/distribution/registry/client/transport/http_reader.go @@ -180,7 +180,6 @@ func (hrs *httpReadSeeker) reader() (io.Reader, error) { // context.GetLogger(hrs.context).Infof("Range: %s", req.Header.Get("Range")) } - req.Header.Add("Accept-Encoding", "identity") resp, err := hrs.client.Do(req) if err != nil { return nil, err diff --git a/vendor/modules.txt b/vendor/modules.txt index e4f7cff5b930d..d8e17fb540ab8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -362,7 +362,7 @@ github.com/deckarep/golang-set/v2 # github.com/dimchansky/utfbom v1.1.1 ## explicit github.com/dimchansky/utfbom -# github.com/docker/distribution v2.8.1+incompatible +# github.com/docker/distribution v2.8.2+incompatible ## explicit github.com/docker/distribution github.com/docker/distribution/digestset From 71846e82c1d14f625fb52ccba168e231df6dbf92 Mon Sep 17 00:00:00 2001 From: Kevin Alvarez Date: Thu, 11 May 2023 15:52:13 +0200 Subject: [PATCH 010/293] bin-image bake target Allows to build a non-runnable image that contains bundles. Signed-off-by: CrazyMax (cherry picked from commit ae1ca671780967a29745c8abbdac0d60eee71655) --- docker-bake.hcl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docker-bake.hcl b/docker-bake.hcl index 899551f9e42e0..26548016d97fd 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -152,6 +152,30 @@ target "all-cross" { inherits = ["all", "_platforms"] } +# +# bin image +# + +target "bin-image" { + inherits = ["all"] + tags = ["moby-bin:local"] + output = ["type=docker"] +} + +target "bin-image-cross" { + inherits = ["bin-image"] + output = ["type=image"] + platforms = [ + "linux/amd64", + "linux/arm/v6", + "linux/arm/v7", + "linux/arm64", + "linux/ppc64le", + "linux/s390x", + "windows/amd64" + ] +} + # # dev # From c76bb6a3a3b8a3a43434749143def0ab8e484fc6 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 11 May 2023 15:45:17 +0200 Subject: [PATCH 011/293] ci: bin-image workflow This workflow will just build the bin-image bake target. Signed-off-by: CrazyMax (cherry picked from commit 135d8f04f9e2845b995a24d343c5cc76dd6efce0) --- .github/workflows/bin-image.yml | 72 +++++++++++++++++++++++++++++++++ docker-bake.hcl | 8 +++- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/bin-image.yml diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml new file mode 100644 index 0000000000000..f8992042cf216 --- /dev/null +++ b/.github/workflows/bin-image.yml @@ -0,0 +1,72 @@ +name: bin-image + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - 'master' + - '[0-9]+.[0-9]+' + tags: + - 'v*' + pull_request: + +env: + PLATFORM: Moby Engine + PRODUCT: Moby + DEFAULT_PRODUCT_LICENSE: Moby + PACKAGER_NAME: Moby + +jobs: + validate-dco: + uses: ./.github/workflows/.dco.yml + + build: + runs-on: ubuntu-20.04 + needs: + - validate-dco + steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: moby-bin + ### versioning strategy + ## push semver tag v23.0.0 + # moby/moby-bin:23.0.0 + # moby/moby-bin:latest + ## push semver prelease tag v23.0.0-beta.1 + # moby/moby-bin:23.0.0-beta.1 + ## push on master + # moby/moby-bin:master + ## push on 23.0 branch + # moby/moby-bin:23.0 + tags: | + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: bin-image-cross + set: | + *.output=type=cacheonly diff --git a/docker-bake.hcl b/docker-bake.hcl index 26548016d97fd..0eb078aebcd6c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -59,6 +59,11 @@ variable "GITHUB_SHA" { default = "" } +# Special target: https://github.com/docker/metadata-action#bake-definition +target "docker-metadata-action" { + tags = ["moby-bin:local"] +} + # Defines the output folder variable "DESTDIR" { default = "" @@ -157,8 +162,7 @@ target "all-cross" { # target "bin-image" { - inherits = ["all"] - tags = ["moby-bin:local"] + inherits = ["all", "docker-metadata-action"] output = ["type=docker"] } From f77a3274b459650a8c762720482b502f9235e6f0 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Tue, 9 May 2023 18:27:40 -0400 Subject: [PATCH 012/293] [chore] clean up reexec.Init() calls Now that most uses of reexec have been replaced with non-reexec solutions, most of the reexec.Init() calls peppered throughout the test suites are unnecessary. Furthermore, most of the reexec.Init() calls in test code neglects to check the return value to determine whether to exit, which would result in the reexec'ed subprocesses proceeding to run the tests, which would reexec another subprocess which would proceed to run the tests, recursively. (That would explain why every reexec callback used to unconditionally call os.Exit() instead of returning...) Remove unneeded reexec.Init() calls from test and example code which no longer needs it, and fix the reexec.Init() calls which are not inert to exit after a reexec callback is invoked. Signed-off-by: Cory Snider (cherry picked from commit 4e0319c87857f180b01f9b072603cc385d7fcee1) Signed-off-by: Sebastiaan van Stijn --- builder/dockerfile/evaluator_test.go | 7 +++++-- builder/remotecontext/tarsum_test.go | 7 +++++-- .../graphdriver/fuse-overlayfs/fuseoverlayfs_test.go | 3 --- daemon/graphdriver/overlay2/overlay_test.go | 3 --- daemon/graphdriver/vfs/vfs_test.go | 6 ------ integration-cli/check_test.go | 3 --- integration/plugin/common/main_test.go | 4 ---- integration/plugin/graphdriver/main_test.go | 5 ----- libnetwork/cmd/readme_test/readme.go | 5 ----- libnetwork/drivers/bridge/port_mapping_test.go | 9 --------- libnetwork/libnetwork_test.go | 11 ----------- libnetwork/osl/sandbox_linux_test.go | 8 -------- pkg/chrootarchive/archive_test.go | 5 ----- 13 files changed, 10 insertions(+), 66 deletions(-) diff --git a/builder/dockerfile/evaluator_test.go b/builder/dockerfile/evaluator_test.go index 03d927061699a..5c40fed55875a 100644 --- a/builder/dockerfile/evaluator_test.go +++ b/builder/dockerfile/evaluator_test.go @@ -21,8 +21,11 @@ type dispatchTestCase struct { files map[string]string } -func init() { - reexec.Init() +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) } func TestDispatch(t *testing.T) { diff --git a/builder/remotecontext/tarsum_test.go b/builder/remotecontext/tarsum_test.go index a398839ab3aaa..ca354d02f9307 100644 --- a/builder/remotecontext/tarsum_test.go +++ b/builder/remotecontext/tarsum_test.go @@ -17,8 +17,11 @@ const ( contents = "contents test" ) -func init() { - reexec.Init() +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) } func TestCloseRootDirectory(t *testing.T) { diff --git a/daemon/graphdriver/fuse-overlayfs/fuseoverlayfs_test.go b/daemon/graphdriver/fuse-overlayfs/fuseoverlayfs_test.go index 0d5b29e53846c..6c0ca01c5eaaf 100644 --- a/daemon/graphdriver/fuse-overlayfs/fuseoverlayfs_test.go +++ b/daemon/graphdriver/fuse-overlayfs/fuseoverlayfs_test.go @@ -9,7 +9,6 @@ import ( "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/daemon/graphdriver/graphtest" "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/reexec" ) func init() { @@ -17,8 +16,6 @@ func init() { // errors or hangs to be debugged directly from the test process. untar = archive.UntarUncompressed graphdriver.ApplyUncompressedLayer = archive.ApplyUncompressedLayer - - reexec.Init() } // This avoids creating a new driver for each test if all tests are run diff --git a/daemon/graphdriver/overlay2/overlay_test.go b/daemon/graphdriver/overlay2/overlay_test.go index 47f3f11005dfa..fbf25fe75bc31 100644 --- a/daemon/graphdriver/overlay2/overlay_test.go +++ b/daemon/graphdriver/overlay2/overlay_test.go @@ -10,7 +10,6 @@ import ( "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/daemon/graphdriver/graphtest" "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/reexec" ) func init() { @@ -18,8 +17,6 @@ func init() { // errors or hangs to be debugged directly from the test process. untar = archive.UntarUncompressed graphdriver.ApplyUncompressedLayer = archive.ApplyUncompressedLayer - - reexec.Init() } func skipIfNaive(t *testing.T) { diff --git a/daemon/graphdriver/vfs/vfs_test.go b/daemon/graphdriver/vfs/vfs_test.go index 63db564518e6c..7cf0bf8cf68cc 100644 --- a/daemon/graphdriver/vfs/vfs_test.go +++ b/daemon/graphdriver/vfs/vfs_test.go @@ -7,14 +7,8 @@ import ( "testing" "github.com/docker/docker/daemon/graphdriver/graphtest" - - "github.com/docker/docker/pkg/reexec" ) -func init() { - reexec.Init() -} - // This avoids creating a new driver for each test if all tests are run // Make sure to put new tests between TestVfsSetup and TestVfsTeardown func TestVfsSetup(t *testing.T) { diff --git a/integration-cli/check_test.go b/integration-cli/check_test.go index 5353672f9c1d2..db6683709ddcf 100644 --- a/integration-cli/check_test.go +++ b/integration-cli/check_test.go @@ -18,7 +18,6 @@ import ( "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration-cli/environment" "github.com/docker/docker/internal/test/suite" - "github.com/docker/docker/pkg/reexec" testdaemon "github.com/docker/docker/testutil/daemon" ienv "github.com/docker/docker/testutil/environment" "github.com/docker/docker/testutil/fakestorage" @@ -50,8 +49,6 @@ var ( func init() { var err error - reexec.Init() // This is required for external graphdriver tests - testEnv, err = environment.New() if err != nil { panic(err) diff --git a/integration/plugin/common/main_test.go b/integration/plugin/common/main_test.go index cd42c8f76153e..5bcbac2a8358a 100644 --- a/integration/plugin/common/main_test.go +++ b/integration/plugin/common/main_test.go @@ -5,16 +5,12 @@ import ( "os" "testing" - "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/testutil/environment" ) var testEnv *environment.Execution func TestMain(m *testing.M) { - if reexec.Init() { - return - } var err error testEnv, err = environment.New() if err != nil { diff --git a/integration/plugin/graphdriver/main_test.go b/integration/plugin/graphdriver/main_test.go index 68fa02c81e9bf..5114180fd7765 100644 --- a/integration/plugin/graphdriver/main_test.go +++ b/integration/plugin/graphdriver/main_test.go @@ -5,7 +5,6 @@ import ( "os" "testing" - "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/testutil/environment" ) @@ -13,10 +12,6 @@ var ( testEnv *environment.Execution ) -func init() { - reexec.Init() // This is required for external graphdriver tests -} - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/libnetwork/cmd/readme_test/readme.go b/libnetwork/cmd/readme_test/readme.go index 7e008a9aa9c0e..80cfae3a6c3ab 100644 --- a/libnetwork/cmd/readme_test/readme.go +++ b/libnetwork/cmd/readme_test/readme.go @@ -8,14 +8,9 @@ import ( "github.com/docker/docker/libnetwork/config" "github.com/docker/docker/libnetwork/netlabel" "github.com/docker/docker/libnetwork/options" - "github.com/docker/docker/pkg/reexec" ) func main() { - if reexec.Init() { - return - } - // Select and configure the network driver networkType := "bridge" diff --git a/libnetwork/drivers/bridge/port_mapping_test.go b/libnetwork/drivers/bridge/port_mapping_test.go index dab375e96b96a..17d6e19b5189a 100644 --- a/libnetwork/drivers/bridge/port_mapping_test.go +++ b/libnetwork/drivers/bridge/port_mapping_test.go @@ -4,23 +4,14 @@ package bridge import ( - "os" "testing" "github.com/docker/docker/libnetwork/netlabel" "github.com/docker/docker/libnetwork/ns" "github.com/docker/docker/libnetwork/testutils" "github.com/docker/docker/libnetwork/types" - "github.com/docker/docker/pkg/reexec" ) -func TestMain(m *testing.M) { - if reexec.Init() { - return - } - os.Exit(m.Run()) -} - func TestPortMappingConfig(t *testing.T) { defer testutils.SetupTestOSContext(t)() d := newDriver() diff --git a/libnetwork/libnetwork_test.go b/libnetwork/libnetwork_test.go index 3707952ab9b27..e3207d4f5c3b8 100644 --- a/libnetwork/libnetwork_test.go +++ b/libnetwork/libnetwork_test.go @@ -10,7 +10,6 @@ import ( "net/http/httptest" "os" "path/filepath" - "runtime" "testing" "github.com/docker/docker/libnetwork" @@ -23,19 +22,9 @@ import ( "github.com/docker/docker/libnetwork/testutils" "github.com/docker/docker/libnetwork/types" "github.com/docker/docker/pkg/plugins" - "github.com/docker/docker/pkg/reexec" - "github.com/sirupsen/logrus" ) func TestMain(m *testing.M) { - if runtime.GOOS == "windows" { - logrus.Info("Test suite does not currently support windows") - os.Exit(0) - } - if reexec.Init() { - return - } - // Cleanup local datastore file _ = os.Remove(datastore.DefaultScope("").Client.Address) diff --git a/libnetwork/osl/sandbox_linux_test.go b/libnetwork/osl/sandbox_linux_test.go index acda5eee9e9c4..0dd8b1e8a570b 100644 --- a/libnetwork/osl/sandbox_linux_test.go +++ b/libnetwork/osl/sandbox_linux_test.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/libnetwork/ns" "github.com/docker/docker/libnetwork/testutils" "github.com/docker/docker/libnetwork/types" - "github.com/docker/docker/pkg/reexec" "github.com/vishvananda/netlink" "github.com/vishvananda/netlink/nl" "github.com/vishvananda/netns" @@ -382,13 +381,6 @@ func TestLiveRestore(t *testing.T) { } } -func TestMain(m *testing.M) { - if reexec.Init() { - return - } - os.Exit(m.Run()) -} - func TestSandboxCreate(t *testing.T) { defer testutils.SetupTestOSContext(t)() diff --git a/pkg/chrootarchive/archive_test.go b/pkg/chrootarchive/archive_test.go index 4e22ac6fd94d2..5ef8ef832bf5b 100644 --- a/pkg/chrootarchive/archive_test.go +++ b/pkg/chrootarchive/archive_test.go @@ -14,14 +14,9 @@ import ( "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/idtools" - "github.com/docker/docker/pkg/reexec" "gotest.tools/v3/skip" ) -func init() { - reexec.Init() -} - var chrootArchiver = NewArchiver(idtools.IdentityMapping{}) func TarUntar(src, dst string) error { From 2b7424512a25b468757bf1d9531a0443253dfa59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 11 May 2023 14:08:14 +0200 Subject: [PATCH 013/293] c8d/authorizer: Default to docker.io MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the `ServerAddress` in the `AuthConfig` provided by the client is empty, default to the default registry (registry-1.docker.io). This makes the behaviour the same as with the containerd image store integration disabled. Signed-off-by: Paweł Gronowski (cherry picked from commit 2ad499f93e4b5117246ceffb8deb2bd6d9966fd2) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/resolver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/containerd/resolver.go b/daemon/containerd/resolver.go index 07c9ed9f0c5a7..9dcfb2a97c873 100644 --- a/daemon/containerd/resolver.go +++ b/daemon/containerd/resolver.go @@ -51,7 +51,7 @@ func hostsWrapper(hostsFn docker.RegistryHosts, authConfig *registrytypes.AuthCo func authorizationCredsFromAuthConfig(authConfig registrytypes.AuthConfig) docker.AuthorizerOpt { cfgHost := registry.ConvertToHostname(authConfig.ServerAddress) - if cfgHost == registry.IndexHostname { + if cfgHost == "" || cfgHost == registry.IndexHostname { cfgHost = registry.DefaultRegistryHost } From 233c49438bd6500b34ea7ad6169f13c39069e48a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 11 May 2023 14:12:49 +0200 Subject: [PATCH 014/293] c8d: Don't create authorizer for empty AuthConfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 3309e45ca19641101dc1a19c4429e96664c97e6c) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/resolver.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/daemon/containerd/resolver.go b/daemon/containerd/resolver.go index 9dcfb2a97c873..5b2d1dff4ccfa 100644 --- a/daemon/containerd/resolver.go +++ b/daemon/containerd/resolver.go @@ -24,7 +24,15 @@ func (i *ImageService) newResolverFromAuthConfig(authConfig *registrytypes.AuthC }), tracker } -func hostsWrapper(hostsFn docker.RegistryHosts, authConfig *registrytypes.AuthConfig, regService RegistryConfigProvider) docker.RegistryHosts { +func hostsWrapper(hostsFn docker.RegistryHosts, optAuthConfig *registrytypes.AuthConfig, regService RegistryConfigProvider) docker.RegistryHosts { + var authorizer docker.Authorizer + if optAuthConfig != nil { + auth := *optAuthConfig + if auth != (registrytypes.AuthConfig{}) { + authorizer = docker.NewDockerAuthorizer(authorizationCredsFromAuthConfig(auth)) + } + } + return func(n string) ([]docker.RegistryHost, error) { hosts, err := hostsFn(n) if err != nil { @@ -33,12 +41,7 @@ func hostsWrapper(hostsFn docker.RegistryHosts, authConfig *registrytypes.AuthCo for i := range hosts { if hosts[i].Authorizer == nil { - var opts []docker.AuthorizerOpt - if authConfig != nil { - opts = append(opts, authorizationCredsFromAuthConfig(*authConfig)) - } - hosts[i].Authorizer = docker.NewDockerAuthorizer(opts...) - + hosts[i].Authorizer = authorizer isInsecure := regService.IsInsecureRegistry(hosts[i].Host) if hosts[i].Client.Transport != nil && isInsecure { hosts[i].Client.Transport = httpFallback{super: hosts[i].Client.Transport} @@ -57,7 +60,10 @@ func authorizationCredsFromAuthConfig(authConfig registrytypes.AuthConfig) docke return docker.WithAuthCreds(func(host string) (string, string, error) { if cfgHost != host { - logrus.WithField("host", host).WithField("cfgHost", cfgHost).Warn("Host doesn't match") + logrus.WithFields(logrus.Fields{ + "host": host, + "cfgHost": cfgHost, + }).Warn("Host doesn't match") return "", "", nil } if authConfig.IdentityToken != "" { From 9717369913214e9fbf1d656af24092d65a1e0102 Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Thu, 13 Apr 2023 13:19:51 +0100 Subject: [PATCH 015/293] c8d: implement classic builder Co-authored-by: Djordje Lukic Signed-off-by: Laura Brehm (cherry picked from commit e46674b6a70dac8f64aed1ab16ccc5e61c335ef5) Signed-off-by: Laura Brehm --- builder/builder.go | 4 +- builder/dockerfile/internals.go | 14 +- builder/dockerfile/internals_test.go | 4 +- builder/dockerfile/mockbackend_test.go | 7 +- daemon/containerd/cache.go | 73 +++- daemon/containerd/image_builder.go | 475 ++++++++++++++++++++++++- daemon/containerd/image_commit.go | 12 +- daemon/containerd/service.go | 3 + daemon/image_service.go | 3 +- daemon/images/image_builder.go | 7 +- 10 files changed, 586 insertions(+), 16 deletions(-) diff --git a/builder/builder.go b/builder/builder.go index 8f33485250783..d3521ddfbbcb5 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -14,6 +14,7 @@ import ( containerpkg "github.com/docker/docker/container" "github.com/docker/docker/image" "github.com/docker/docker/layer" + "github.com/opencontainers/go-digest" ) const ( @@ -45,7 +46,7 @@ type Backend interface { // ContainerCreateWorkdir creates the workdir ContainerCreateWorkdir(containerID string) error - CreateImage(config []byte, parent string) (Image, error) + CreateImage(ctx context.Context, config []byte, parent string, contentStoreDigest digest.Digest) (Image, error) ImageCacheBuilder } @@ -104,6 +105,7 @@ type ROLayer interface { Release() error NewRWLayer() (RWLayer, error) DiffID() layer.DiffID + ContentStoreDigest() digest.Digest } // RWLayer is active layer that can be read/modified diff --git a/builder/dockerfile/internals.go b/builder/dockerfile/internals.go index 66d873e2e5f9c..6aa96a972cd92 100644 --- a/builder/dockerfile/internals.go +++ b/builder/dockerfile/internals.go @@ -63,7 +63,7 @@ func (b *Builder) commitContainer(ctx context.Context, dispatchState *dispatchSt return err } -func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error { +func (b *Builder) exportImage(ctx context.Context, state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error { newLayer, err := layer.Commit() if err != nil { return err @@ -98,7 +98,15 @@ func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, paren return errors.Wrap(err, "failed to encode image config") } - exportedImage, err := b.docker.CreateImage(config, state.imageID) + // when writing the new image's manifest, we now need to pass in the new layer's digest. + // before the containerd store work this was unnecessary since we get the layer id + // from the image's RootFS ChainID -- see: + // https://github.com/moby/moby/blob/8cf66ed7322fa885ef99c4c044fa23e1727301dc/image/store.go#L162 + // however, with the containerd store we can't do this. An alternative implementation here + // without changing the signature would be to get the layer digest by walking the content store + // and filtering the objects to find the layer with the DiffID we want, but that has performance + // implications that should be called out/investigated + exportedImage, err := b.docker.CreateImage(ctx, config, state.imageID, newLayer.ContentStoreDigest()) if err != nil { return errors.Wrapf(err, "failed to export image") } @@ -170,7 +178,7 @@ func (b *Builder) performCopy(ctx context.Context, req dispatchRequest, inst cop return errors.Wrapf(err, "failed to copy files") } } - return b.exportImage(state, rwLayer, imageMount.Image(), runConfigWithCommentCmd) + return b.exportImage(ctx, state, rwLayer, imageMount.Image(), runConfigWithCommentCmd) } func createDestInfo(workingDir string, inst copyInstruction, rwLayer builder.RWLayer, platform string) (copyInfo, error) { diff --git a/builder/dockerfile/internals_test.go b/builder/dockerfile/internals_test.go index 8145fac90d091..f56c44b751cb8 100644 --- a/builder/dockerfile/internals_test.go +++ b/builder/dockerfile/internals_test.go @@ -1,6 +1,7 @@ package dockerfile // import "github.com/docker/docker/builder/dockerfile" import ( + "context" "fmt" "os" "runtime" @@ -193,6 +194,7 @@ type MockROLayer struct { diffID layer.DiffID } +func (l *MockROLayer) ContentStoreDigest() digest.Digest { return "" } func (l *MockROLayer) Release() error { return nil } func (l *MockROLayer) NewRWLayer() (builder.RWLayer, error) { return nil, nil } func (l *MockROLayer) DiffID() layer.DiffID { return l.diffID } @@ -217,6 +219,6 @@ func TestExportImage(t *testing.T) { imageSources: getMockImageSource(nil, nil, nil), docker: getMockBuildBackend(), } - err := b.exportImage(ds, layer, parentImage, runConfig) + err := b.exportImage(context.TODO(), ds, layer, parentImage, runConfig) assert.NilError(t, err) } diff --git a/builder/dockerfile/mockbackend_test.go b/builder/dockerfile/mockbackend_test.go index 5b5419cfd6a5d..a9e43e9bd1f1c 100644 --- a/builder/dockerfile/mockbackend_test.go +++ b/builder/dockerfile/mockbackend_test.go @@ -13,6 +13,7 @@ import ( containerpkg "github.com/docker/docker/container" "github.com/docker/docker/image" "github.com/docker/docker/layer" + "github.com/opencontainers/go-digest" ) // MockBackend implements the builder.Backend interface for unit testing @@ -80,7 +81,7 @@ func (m *MockBackend) MakeImageCache(ctx context.Context, cacheFrom []string) (b return nil, nil } -func (m *MockBackend) CreateImage(config []byte, parent string) (builder.Image, error) { +func (m *MockBackend) CreateImage(ctx context.Context, config []byte, parent string, layerDigest digest.Digest) (builder.Image, error) { return &mockImage{id: "test"}, nil } @@ -119,6 +120,10 @@ func (mic *mockImageCache) GetCache(parentID string, cfg *container.Config) (str type mockLayer struct{} +func (l *mockLayer) ContentStoreDigest() digest.Digest { + return "" +} + func (l *mockLayer) Release() error { return nil } diff --git a/daemon/containerd/cache.go b/daemon/containerd/cache.go index e066e8ae879be..e9fa697932c71 100644 --- a/daemon/containerd/cache.go +++ b/daemon/containerd/cache.go @@ -2,11 +2,82 @@ package containerd import ( "context" + "reflect" + "github.com/docker/docker/api/types/container" + imagetype "github.com/docker/docker/api/types/image" "github.com/docker/docker/builder" + "github.com/docker/docker/image" ) // MakeImageCache creates a stateful image cache. func (i *ImageService) MakeImageCache(ctx context.Context, cacheFrom []string) (builder.ImageCache, error) { - panic("not implemented") + images := []*image.Image{} + for _, c := range cacheFrom { + im, err := i.GetImage(ctx, c, imagetype.GetImageOpts{}) + if err != nil { + return nil, err + } + images = append(images, im) + } + return &imageCache{images: images, c: i}, nil +} + +type imageCache struct { + images []*image.Image + c *ImageService +} + +func (ic *imageCache) GetCache(parentID string, cfg *container.Config) (imageID string, err error) { + ctx := context.TODO() + cfgCpy := *cfg + i, err := ic.c.GetImage(ctx, parentID, imagetype.GetImageOpts{}) + if err != nil { + for _, ii := range ic.images { + if ii.ID().String() == parentID { + if compare(ii.RunConfig(), &cfgCpy) { + return ii.ID().String(), nil + } + } + } + } else { + children, err := ic.c.Children(ctx, i.ID()) + if err != nil { + return "", err + } + for _, ch := range children { + childImage, err := ic.c.GetImage(context.TODO(), ch.String(), imagetype.GetImageOpts{}) + if err != nil { + return "", err + } + // this implementation looks correct but it's currently not working + // with the containerd store as we're not storing the image ContainerConfig + // and so intermediate images with ContainerConfigs such as + // #(nop) COPY file:c6ab44934e83eeb07289a211582c6faa25dea7d06dae077b6ef76029e92400ce in ... + // are not getting a hit + if compare(&childImage.ContainerConfig, &cfgCpy) { + return ch.String(), nil + } + } + } + return "", nil +} + +// compare two Config structs. Do not consider the "Hostname" field as it +// defaults to the randomly generated short container ID or "Image" as it +// represents the name of the image as it was passed in (symbolic) and does +// not provide any meaningful information about whether the image is usable +// as cache. +// If OpenStdin is set, then it differs +func compare(a, b *container.Config) bool { + if a == nil || b == nil || + a.OpenStdin || b.OpenStdin { + return false + } + + a.Image = "" + a.Hostname = "" + b.Image = "" + b.Hostname = "" + return reflect.DeepEqual(a, b) } diff --git a/daemon/containerd/image_builder.go b/daemon/containerd/image_builder.go index e70ec9d6c5a6c..53adde38ad7cd 100644 --- a/daemon/containerd/image_builder.go +++ b/daemon/containerd/image_builder.go @@ -2,23 +2,490 @@ package containerd import ( "context" - "errors" + "fmt" + "io" + "os" + "runtime" + "time" + "github.com/containerd/containerd" + cerrdefs "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/leases" + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/rootfs" + "github.com/docker/distribution/reference" "github.com/docker/docker/api/types/backend" + imagetypes "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/registry" + registrypkg "github.com/docker/docker/registry" + + // "github.com/docker/docker/api/types/container" + containerdimages "github.com/containerd/containerd/images" + "github.com/docker/docker/api/types/image" "github.com/docker/docker/builder" "github.com/docker/docker/errdefs" + dimage "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/progress" + "github.com/docker/docker/pkg/streamformatter" + "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/pkg/system" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" ) // GetImageAndReleasableLayer returns an image and releaseable layer for a // reference or ID. Every call to GetImageAndReleasableLayer MUST call // releasableLayer.Release() to prevent leaking of layers. func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { - return nil, nil, errdefs.NotImplemented(errors.New("not implemented")) + if refOrID == "" { // from SCRATCH + os := runtime.GOOS + if runtime.GOOS == "windows" { + os = "linux" + } + if opts.Platform != nil { + os = opts.Platform.OS + } + if !system.IsOSSupported(os) { + return nil, nil, system.ErrNotSupportedOperatingSystem + } + return nil, &rolayer{ + key: "", + c: i.client, + snapshotter: i.snapshotter, + diffID: "", + root: "", + }, nil + } + + if opts.PullOption != backend.PullOptionForcePull { + // TODO(laurazard): same as below + img, err := i.GetImage(ctx, refOrID, image.GetImageOpts{Platform: opts.Platform}) + if err != nil && opts.PullOption == backend.PullOptionNoPull { + return nil, nil, err + } + imgDesc, err := i.resolveDescriptor(ctx, refOrID) + if err != nil && !errdefs.IsNotFound(err) { + return nil, nil, err + } + if img != nil { + if !system.IsOSSupported(img.OperatingSystem()) { + return nil, nil, system.ErrNotSupportedOperatingSystem + } + + layer, err := newROLayerForImage(ctx, &imgDesc, i, opts, refOrID, opts.Platform) + if err != nil { + return nil, nil, err + } + + return img, layer, nil + } + } + + ctx, _, err := i.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) + if err != nil { + return nil, nil, fmt.Errorf("failed to create lease for commit: %w", err) + } + + // TODO(laurazard): do we really need a new method here to pull the image? + imgDesc, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform) + if err != nil { + return nil, nil, err + } + + // TODO(laurazard): pullForBuilder should return whatever we + // need here instead of having to go and get it again + img, err := i.GetImage(ctx, refOrID, imagetypes.GetImageOpts{ + Platform: opts.Platform, + }) + if err != nil { + return nil, nil, err + } + + layer, err := newROLayerForImage(ctx, imgDesc, i, opts, refOrID, opts.Platform) + if err != nil { + return nil, nil, err + } + + return img, layer, nil +} + +func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]registry.AuthConfig, output io.Writer, platform *ocispec.Platform) (*ocispec.Descriptor, error) { + ref, err := reference.ParseNormalizedNamed(name) + if err != nil { + return nil, err + } + taggedRef := reference.TagNameOnly(ref) + + pullRegistryAuth := ®istry.AuthConfig{} + if len(authConfigs) > 0 { + // The request came with a full auth config, use it + repoInfo, err := i.registryService.ResolveRepository(ref) + if err != nil { + return nil, err + } + + resolvedConfig := registrypkg.ResolveAuthConfig(authConfigs, repoInfo.Index) + pullRegistryAuth = &resolvedConfig + } + + if err := i.PullImage(ctx, ref.Name(), taggedRef.(reference.NamedTagged).Tag(), platform, nil, pullRegistryAuth, output); err != nil { + return nil, err + } + + img, err := i.GetImage(ctx, name, imagetypes.GetImageOpts{Platform: platform}) + if err != nil { + if errdefs.IsNotFound(err) && img != nil && platform != nil { + imgPlat := ocispec.Platform{ + OS: img.OS, + Architecture: img.BaseImgArch(), + Variant: img.BaseImgVariant(), + } + + p := *platform + if !platforms.Only(p).Match(imgPlat) { + po := streamformatter.NewJSONProgressOutput(output, false) + progress.Messagef(po, "", ` +WARNING: Pulled image with specified platform (%s), but the resulting image's configured platform (%s) does not match. +This is most likely caused by a bug in the build system that created the fetched image (%s). +Please notify the image author to correct the configuration.`, + platforms.Format(p), platforms.Format(imgPlat), name, + ) + logrus.WithError(err).WithField("image", name).Warn("Ignoring error about platform mismatch where the manifest list points to an image whose configuration does not match the platform in the manifest.") + } + } else { + return nil, err + } + } + + if !system.IsOSSupported(img.OperatingSystem()) { + return nil, system.ErrNotSupportedOperatingSystem + } + + imgDesc, err := i.resolveDescriptor(ctx, name) + if err != nil { + return nil, err + } + + return &imgDesc, err +} + +func newROLayerForImage(ctx context.Context, imgDesc *ocispec.Descriptor, i *ImageService, opts backend.GetImageAndLayerOptions, refOrID string, platform *ocispec.Platform) (builder.ROLayer, error) { + if imgDesc == nil { + return nil, fmt.Errorf("can't make an RO layer for a nil image :'(") + } + + platMatcher := platforms.Default() + if platform != nil { + platMatcher = platforms.Only(*platform) + } + + // this needs it's own context + lease so that it doesn't get cleaned before we're ready + confDesc, err := containerdimages.Config(ctx, i.client.ContentStore(), *imgDesc, platMatcher) + if err != nil { + return nil, err + } + + diffIDs, err := containerdimages.RootFS(ctx, i.client.ContentStore(), confDesc) + if err != nil { + return nil, err + } + parent := identity.ChainID(diffIDs).String() + + s := i.client.SnapshotService(i.snapshotter) + key := stringid.GenerateRandomID() + ctx, _, err = i.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) + if err != nil { + return nil, fmt.Errorf("failed to create lease for commit: %w", err) + } + mounts, err := s.View(ctx, key, parent) + if err != nil { + return nil, err + } + + tempMountLocation := os.TempDir() + root, err := os.MkdirTemp(tempMountLocation, "rootfs-mount") + if err != nil { + return nil, err + } + + if err := mount.All(mounts, root); err != nil { + return nil, err + } + + return &rolayer{ + key: key, + c: i.client, + snapshotter: i.snapshotter, + diffID: digest.Digest(parent), + root: root, + contentStoreDigest: "", + }, nil +} + +type rolayer struct { + key string + c *containerd.Client + snapshotter string + diffID digest.Digest + root string + contentStoreDigest digest.Digest +} + +func (rl *rolayer) ContentStoreDigest() digest.Digest { + return rl.contentStoreDigest +} + +func (rl *rolayer) DiffID() layer.DiffID { + if rl.diffID == "" { + return layer.DigestSHA256EmptyTar + } + return layer.DiffID(rl.diffID) +} + +func (rl *rolayer) Release() error { + snapshotter := rl.c.SnapshotService(rl.snapshotter) + err := snapshotter.Remove(context.TODO(), rl.key) + if err != nil && !cerrdefs.IsNotFound(err) { + return err + } + + if rl.root == "" { // nothing to release + return nil + } + if err := mount.UnmountAll(rl.root, 0); err != nil { + logrus.WithError(err).WithField("root", rl.root).Error("failed to unmount ROLayer") + return err + } + if err := os.Remove(rl.root); err != nil { + logrus.WithError(err).WithField("dir", rl.root).Error("failed to remove mount temp dir") + return err + } + rl.root = "" + return nil +} + +// NewRWLayer creates a new read-write layer for the builder +func (rl *rolayer) NewRWLayer() (builder.RWLayer, error) { + snapshotter := rl.c.SnapshotService(rl.snapshotter) + + // we need this here for the prepared snapshots or + // we'll have racy behaviour where sometimes they + // will get GC'd before we commit/use them + ctx, _, err := rl.c.WithLease(context.TODO(), leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) + if err != nil { + return nil, fmt.Errorf("failed to create lease for commit: %w", err) + } + + key := stringid.GenerateRandomID() + mounts, err := snapshotter.Prepare(ctx, key, rl.diffID.String()) + if err != nil { + return nil, err + } + + root, err := os.MkdirTemp(os.TempDir(), "rootfs-mount") + if err != nil { + return nil, err + } + if err := mount.All(mounts, root); err != nil { + return nil, err + } + + return &rwlayer{ + key: key, + parent: rl.key, + c: rl.c, + snapshotter: rl.snapshotter, + root: root, + }, nil +} + +type rwlayer struct { + key string + parent string + c *containerd.Client + snapshotter string + root string +} + +func (rw *rwlayer) Root() string { + return rw.root +} + +func (rw *rwlayer) Commit() (builder.ROLayer, error) { + // we need this here for the prepared snapshots or + // we'll have racy behaviour where sometimes they + // will get GC'd before we commit/use them + ctx, _, err := rw.c.WithLease(context.TODO(), leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) + if err != nil { + return nil, fmt.Errorf("failed to create lease for commit: %w", err) + } + snapshotter := rw.c.SnapshotService(rw.snapshotter) + + key := stringid.GenerateRandomID() + err = snapshotter.Commit(ctx, key, rw.key) + if err != nil && !cerrdefs.IsAlreadyExists(err) { + return nil, err + } + + differ := rw.c.DiffService() + desc, err := rootfs.CreateDiff(ctx, key, snapshotter, differ) + if err != nil { + return nil, err + } + info, err := rw.c.ContentStore().Info(ctx, desc.Digest) + if err != nil { + return nil, err + } + diffIDStr, ok := info.Labels["containerd.io/uncompressed"] + if !ok { + return nil, fmt.Errorf("invalid differ response with no diffID") + } + diffID, err := digest.Parse(diffIDStr) + if err != nil { + return nil, err + } + + return &rolayer{ + key: key, + c: rw.c, + snapshotter: rw.snapshotter, + diffID: diffID, + root: "", + contentStoreDigest: desc.Digest, + }, nil +} + +func (rw *rwlayer) Release() error { + snapshotter := rw.c.SnapshotService(rw.snapshotter) + err := snapshotter.Remove(context.TODO(), rw.key) + if err != nil && !cerrdefs.IsNotFound(err) { + return err + } + + if rw.root == "" { // nothing to release + return nil + } + if err := mount.UnmountAll(rw.root, 0); err != nil { + logrus.WithError(err).WithField("root", rw.root).Error("failed to unmount ROLayer") + return err + } + if err := os.Remove(rw.root); err != nil { + logrus.WithError(err).WithField("dir", rw.root).Error("failed to remove mount temp dir") + return err + } + rw.root = "" + return nil } // CreateImage creates a new image by adding a config and ID to the image store. // This is similar to LoadImage() except that it receives JSON encoded bytes of // an image instead of a tar archive. -func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) { - return nil, errdefs.NotImplemented(errors.New("not implemented")) +func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent string, layerDigest digest.Digest) (builder.Image, error) { + imgToCreate, err := dimage.NewFromJSON(config) + if err != nil { + return nil, err + } + + rootfs := ocispec.RootFS{ + Type: imgToCreate.RootFS.Type, + DiffIDs: []digest.Digest{}, + } + for _, diffId := range imgToCreate.RootFS.DiffIDs { + rootfs.DiffIDs = append(rootfs.DiffIDs, digest.Digest(diffId)) + } + exposedPorts := make(map[string]struct{}, len(imgToCreate.Config.ExposedPorts)) + for k, v := range imgToCreate.Config.ExposedPorts { + exposedPorts[string(k)] = v + } + + // make an ocispec.Image from the docker/image.Image + ociImgToCreate := ocispec.Image{ + Created: &imgToCreate.Created, + Author: imgToCreate.Author, + Architecture: imgToCreate.Architecture, + Variant: imgToCreate.Variant, + OS: imgToCreate.OS, + OSVersion: imgToCreate.OSVersion, + OSFeatures: imgToCreate.OSFeatures, + Config: ocispec.ImageConfig{ + User: imgToCreate.Config.User, + ExposedPorts: exposedPorts, + Env: imgToCreate.Config.Env, + Entrypoint: imgToCreate.Config.Entrypoint, + Cmd: imgToCreate.Config.Cmd, + Volumes: imgToCreate.Config.Volumes, + WorkingDir: imgToCreate.Config.WorkingDir, + Labels: imgToCreate.Config.Labels, + StopSignal: imgToCreate.Config.StopSignal, + }, + RootFS: rootfs, + // TODO(laurazard) + History: []ocispec.History{}, + } + + var layers []ocispec.Descriptor + // if the image has a parent, we need to start with the parents layers descriptors + if parent != "" { + parentDesc, err := i.resolveDescriptor(ctx, parent) + if err != nil { + return nil, err + } + parentImageManifest, err := containerdimages.Manifest(ctx, i.client.ContentStore(), parentDesc, platforms.Default()) + if err != nil { + return nil, err + } + + layers = parentImageManifest.Layers + } + + // get the info for the new layers + info, err := i.client.ContentStore().Info(ctx, layerDigest) + if err != nil { + return nil, err + } + + // append the new layer descriptor + layers = append(layers, + ocispec.Descriptor{ + MediaType: containerdimages.MediaTypeDockerSchema2LayerGzip, + Digest: layerDigest, + Size: info.Size, + }, + ) + + commitManifestDesc, err := writeContentsForImage(ctx, i.snapshotter, i.client.ContentStore(), ociImgToCreate, layers) + if err != nil { + return nil, err + } + + // image create + img := containerdimages.Image{ + Name: danglingImageName(commitManifestDesc.Digest), + Target: commitManifestDesc, + CreatedAt: time.Now(), + } + + createdImage, err := i.client.ImageService().Update(ctx, img) + if err != nil { + if !cerrdefs.IsNotFound(err) { + return nil, err + } + + if createdImage, err = i.client.ImageService().Create(ctx, img); err != nil { + return nil, fmt.Errorf("failed to create new image: %w", err) + } + } + + if err := i.unpackImage(ctx, createdImage, platforms.DefaultSpec()); err != nil { + return nil, err + } + + newImage := dimage.NewImage(dimage.ID(createdImage.Target.Digest)) + newImage.V1Image = imgToCreate.V1Image + newImage.V1Image.ID = string(createdImage.Target.Digest) + return newImage, nil } diff --git a/daemon/containerd/image_commit.go b/daemon/containerd/image_commit.go index bbb4df0ddc467..a721c41970ece 100644 --- a/daemon/containerd/image_commit.go +++ b/daemon/containerd/image_commit.go @@ -6,7 +6,6 @@ import ( "crypto/rand" "encoding/base64" "encoding/json" - "errors" "fmt" "runtime" "strings" @@ -20,7 +19,6 @@ import ( "github.com/containerd/containerd/rootfs" "github.com/containerd/containerd/snapshots" "github.com/docker/docker/api/types/backend" - "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" @@ -298,5 +296,13 @@ func uniquePart() string { // // This is a temporary shim. Should be removed when builder stops using commit. func (i *ImageService) CommitBuildStep(ctx context.Context, c backend.CommitConfig) (image.ID, error) { - return "", errdefs.NotImplemented(errors.New("not implemented")) + ctr := i.containers.Get(c.ContainerID) + if ctr == nil { + // TODO: use typed error + return "", fmt.Errorf("container not found: %s", c.ContainerID) + } + c.ContainerMountLabel = ctr.MountLabel + c.ContainerOS = ctr.OS + c.ParentImageID = string(ctr.ImageID) + return i.CommitImage(ctx, c) } diff --git a/daemon/containerd/service.go b/daemon/containerd/service.go index d46bbd8c53acb..0aa0c39146773 100644 --- a/daemon/containerd/service.go +++ b/daemon/containerd/service.go @@ -10,6 +10,7 @@ import ( "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/snapshots" + "github.com/docker/distribution/reference" imagetypes "github.com/docker/docker/api/types/image" "github.com/docker/docker/container" daemonevents "github.com/docker/docker/daemon/events" @@ -17,6 +18,7 @@ import ( "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/layer" + "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -41,6 +43,7 @@ type RegistryHostsProvider interface { type RegistryConfigProvider interface { IsInsecureRegistry(host string) bool + ResolveRepository(name reference.Named) (*registry.RepositoryInfo, error) } type ImageServiceConfig struct { diff --git a/daemon/image_service.go b/daemon/image_service.go index b47575b6db9ba..8470d18e025d8 100644 --- a/daemon/image_service.go +++ b/daemon/image_service.go @@ -16,6 +16,7 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/archive" + "github.com/opencontainers/go-digest" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -27,7 +28,7 @@ type ImageService interface { PullImage(ctx context.Context, name, tag string, platform *v1.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error PushImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error - CreateImage(config []byte, parent string) (builder.Image, error) + CreateImage(ctx context.Context, config []byte, parent string, contentStoreDigest digest.Digest) (builder.Image, error) ImageDelete(ctx context.Context, imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) ExportImage(ctx context.Context, names []string, outStream io.Writer) error PerformWithBaseFS(ctx context.Context, c *container.Container, fn func(string) error) error diff --git a/daemon/images/image_builder.go b/daemon/images/image_builder.go index af2d4073ccd7d..b10878f9f1420 100644 --- a/daemon/images/image_builder.go +++ b/daemon/images/image_builder.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/system" registrypkg "github.com/docker/docker/registry" + "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -30,6 +31,10 @@ type roLayer struct { roLayer layer.Layer } +func (l *roLayer) ContentStoreDigest() digest.Digest { + return "" +} + func (l *roLayer) DiffID() layer.DiffID { if l.roLayer == nil { return layer.DigestSHA256EmptyTar @@ -241,7 +246,7 @@ func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s // CreateImage creates a new image by adding a config and ID to the image store. // This is similar to LoadImage() except that it receives JSON encoded bytes of // an image instead of a tar archive. -func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) { +func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent string, _ digest.Digest) (builder.Image, error) { id, err := i.imageStore.Create(config) if err != nil { return nil, errors.Wrapf(err, "failed to create image") From 8587a1c617dae69474e8e3847d5fd799de156da6 Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Mon, 8 May 2023 23:51:40 +0100 Subject: [PATCH 016/293] c8d/builder: implement cache Signed-off-by: Laura Brehm (cherry picked from commit bd6868557d7ceb58d1d4717737e9da755cad87e5) Signed-off-by: Laura Brehm --- daemon/containerd/cache.go | 78 ++++++++++++++++-------------- daemon/containerd/image_builder.go | 26 ++++++++-- daemon/containerd/image_commit.go | 9 ++-- 3 files changed, 69 insertions(+), 44 deletions(-) diff --git a/daemon/containerd/cache.go b/daemon/containerd/cache.go index e9fa697932c71..8bd01768f8f9a 100644 --- a/daemon/containerd/cache.go +++ b/daemon/containerd/cache.go @@ -3,6 +3,7 @@ package containerd import ( "context" "reflect" + "strings" "github.com/docker/docker/api/types/container" imagetype "github.com/docker/docker/api/types/image" @@ -30,54 +31,57 @@ type imageCache struct { func (ic *imageCache) GetCache(parentID string, cfg *container.Config) (imageID string, err error) { ctx := context.TODO() - cfgCpy := *cfg - i, err := ic.c.GetImage(ctx, parentID, imagetype.GetImageOpts{}) + parent, err := ic.c.GetImage(ctx, parentID, imagetype.GetImageOpts{}) if err != nil { - for _, ii := range ic.images { - if ii.ID().String() == parentID { - if compare(ii.RunConfig(), &cfgCpy) { - return ii.ID().String(), nil - } - } + return "", err + } + + for _, localCachedImage := range ic.images { + if isMatch(localCachedImage, parent, cfg) { + return localCachedImage.ID().String(), nil } - } else { - children, err := ic.c.Children(ctx, i.ID()) + } + + children, err := ic.c.Children(ctx, parent.ID()) + if err != nil { + return "", err + } + + for _, children := range children { + childImage, err := ic.c.GetImage(ctx, children.String(), imagetype.GetImageOpts{}) if err != nil { return "", err } - for _, ch := range children { - childImage, err := ic.c.GetImage(context.TODO(), ch.String(), imagetype.GetImageOpts{}) - if err != nil { - return "", err - } - // this implementation looks correct but it's currently not working - // with the containerd store as we're not storing the image ContainerConfig - // and so intermediate images with ContainerConfigs such as - // #(nop) COPY file:c6ab44934e83eeb07289a211582c6faa25dea7d06dae077b6ef76029e92400ce in ... - // are not getting a hit - if compare(&childImage.ContainerConfig, &cfgCpy) { - return ch.String(), nil - } + + if isMatch(childImage, parent, cfg) { + return children.String(), nil } } + return "", nil } -// compare two Config structs. Do not consider the "Hostname" field as it -// defaults to the randomly generated short container ID or "Image" as it -// represents the name of the image as it was passed in (symbolic) and does -// not provide any meaningful information about whether the image is usable -// as cache. -// If OpenStdin is set, then it differs -func compare(a, b *container.Config) bool { - if a == nil || b == nil || - a.OpenStdin || b.OpenStdin { +// isMatch checks whether a given target can be used as cache for the given +// parent image/config combination. +// A target can only be an immediate child of the given parent image. For +// a parent image with `n` history entries, a valid target must have `n+1` +// entries and the extra entry must match the provided config +func isMatch(target, parent *image.Image, cfg *container.Config) bool { + if target == nil || parent == nil || cfg == nil { + return false + } + + if len(target.History) != len(parent.History)+1 || + len(target.RootFS.DiffIDs) != len(parent.RootFS.DiffIDs)+1 { return false } - a.Image = "" - a.Hostname = "" - b.Image = "" - b.Hostname = "" - return reflect.DeepEqual(a, b) + for i := range parent.History { + if !reflect.DeepEqual(parent.History[i], target.History[i]) { + return false + } + } + + childCreatedBy := target.History[len(target.History)-1].CreatedBy + return childCreatedBy == strings.Join(cfg.Cmd, " ") } diff --git a/daemon/containerd/image_builder.go b/daemon/containerd/image_builder.go index 53adde38ad7cd..c92bd8bf3e88f 100644 --- a/daemon/containerd/image_builder.go +++ b/daemon/containerd/image_builder.go @@ -402,6 +402,18 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st exposedPorts[string(k)] = v } + var ociHistory []ocispec.History + for _, history := range imgToCreate.History { + created := history.Created + ociHistory = append(ociHistory, ocispec.History{ + Created: &created, + CreatedBy: history.CreatedBy, + Author: history.Author, + Comment: history.Comment, + EmptyLayer: history.EmptyLayer, + }) + } + // make an ocispec.Image from the docker/image.Image ociImgToCreate := ocispec.Image{ Created: &imgToCreate.Created, @@ -422,9 +434,8 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st Labels: imgToCreate.Config.Labels, StopSignal: imgToCreate.Config.StopSignal, }, - RootFS: rootfs, - // TODO(laurazard) - History: []ocispec.History{}, + RootFS: rootfs, + History: ociHistory, } var layers []ocispec.Descriptor @@ -457,6 +468,14 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st }, ) + // necessary to prevent the contents from being GC'd + // between writing them here and creating an image + ctx, done, err := i.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) + if err != nil { + return nil, err + } + defer done(ctx) + commitManifestDesc, err := writeContentsForImage(ctx, i.snapshotter, i.client.ContentStore(), ociImgToCreate, layers) if err != nil { return nil, err @@ -487,5 +506,6 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st newImage := dimage.NewImage(dimage.ID(createdImage.Target.Digest)) newImage.V1Image = imgToCreate.V1Image newImage.V1Image.ID = string(createdImage.Target.Digest) + newImage.History = imgToCreate.History return newImage, nil } diff --git a/daemon/containerd/image_commit.go b/daemon/containerd/image_commit.go index a721c41970ece..632f28bc6bf5b 100644 --- a/daemon/containerd/image_commit.go +++ b/daemon/containerd/image_commit.go @@ -141,10 +141,11 @@ func generateCommitImageConfig(baseConfig ocispec.Image, diffID digest.Digest, o DiffIDs: append(baseConfig.RootFS.DiffIDs, diffID), }, History: append(baseConfig.History, ocispec.History{ - Created: &createdTime, - CreatedBy: strings.Join(opts.ContainerConfig.Cmd, " "), - Author: opts.Author, - Comment: opts.Comment, + Created: &createdTime, + CreatedBy: strings.Join(opts.ContainerConfig.Cmd, " "), + Author: opts.Author, + Comment: opts.Comment, + // TODO(laurazard): this check might be incorrect EmptyLayer: diffID == "", }), } From 8bbfa3274109a240952c08a2263d0cd256a27311 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Fri, 12 May 2023 09:33:10 +0200 Subject: [PATCH 017/293] c8d: The authorizer needs to be set even if AuthConfig is empty Without the authorizer pulling will fail if the user is not logged-in Signed-off-by: Djordje Lukic --- daemon/containerd/resolver.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/daemon/containerd/resolver.go b/daemon/containerd/resolver.go index 5b2d1dff4ccfa..5edd83181b19f 100644 --- a/daemon/containerd/resolver.go +++ b/daemon/containerd/resolver.go @@ -27,10 +27,7 @@ func (i *ImageService) newResolverFromAuthConfig(authConfig *registrytypes.AuthC func hostsWrapper(hostsFn docker.RegistryHosts, optAuthConfig *registrytypes.AuthConfig, regService RegistryConfigProvider) docker.RegistryHosts { var authorizer docker.Authorizer if optAuthConfig != nil { - auth := *optAuthConfig - if auth != (registrytypes.AuthConfig{}) { - authorizer = docker.NewDockerAuthorizer(authorizationCredsFromAuthConfig(auth)) - } + authorizer = docker.NewDockerAuthorizer(authorizationCredsFromAuthConfig(*optAuthConfig)) } return func(n string) ([]docker.RegistryHost, error) { From 907f037141b41d0a96daa379fd2cbd0b0eee7569 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 15 May 2023 12:52:50 +0100 Subject: [PATCH 018/293] update containerd binary to v1.7.1 full diff: https://github.com/containerd/containerd/compare/v1.7.0...v1.7.1 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 484785456cd81da9e97d0f5ef1aaf33be4d7a991) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- Dockerfile.windows | 2 +- hack/dockerfile/install/containerd.installer | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a610a45e9d0d0..2ffcb830d966a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -192,7 +192,7 @@ RUN git init . && git remote add origin "https://github.com/containerd/container # When updating the binary version you may also need to update the vendor # version to pick up bug fixes or new APIs, however, usually the Go packages # are built from a commit from the master branch. -ARG CONTAINERD_VERSION=v1.7.0 +ARG CONTAINERD_VERSION=v1.7.1 RUN git fetch -q --depth 1 origin "${CONTAINERD_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD FROM base AS containerd-build diff --git a/Dockerfile.windows b/Dockerfile.windows index c3ecc56ef9e05..43258035d25bf 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -168,7 +168,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref ARG GO_VERSION=1.20.4 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 -ARG CONTAINERD_VERSION=v1.7.0 +ARG CONTAINERD_VERSION=v1.7.1 # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. diff --git a/hack/dockerfile/install/containerd.installer b/hack/dockerfile/install/containerd.installer index bd150ca10a4e5..bfdd27612fe45 100755 --- a/hack/dockerfile/install/containerd.installer +++ b/hack/dockerfile/install/containerd.installer @@ -15,7 +15,7 @@ set -e # the binary version you may also need to update the vendor version to pick up # bug fixes or new APIs, however, usually the Go packages are built from a # commit from the master branch. -: "${CONTAINERD_VERSION:=v1.7.0}" +: "${CONTAINERD_VERSION:=v1.7.1}" install_containerd() ( echo "Install containerd version $CONTAINERD_VERSION" From 5ea7b8d091091cb33463c8829de9c609720e2e8a Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Thu, 18 May 2023 13:14:59 +0100 Subject: [PATCH 019/293] fix: `docker pull` with platform checks wrong image tag This fixes a bug where, if a user pulls an image with a tag != `latest` and a specific platform, we return an NotFound error for the wrong (`latest`) tag. see: https://github.com/moby/moby/issues/45558 This bug was introduced in https://github.com/moby/moby/commit/779a5b3029473c6bb61c2d08a43e0a8d68a73d7f in the changes to `daemon/images/image_pull.go`, when we started returning the error from the call to `GetImage` after the pull. We do this call, if pulling with a specified platform, to check if the platform of the pulled image matches the requested platform (for cases with single-arch images). However, when we call `GetImage` we're not passing the image tag, only name, so `GetImage` assumes `latest` which breaks when the user has requested a different tag, since there might not be such an image in the store. Signed-off-by: Laura Brehm (cherry picked from commit f450ea64e6b3667496a2d315002fa7f58a2d2477) Signed-off-by: Laura Brehm --- daemon/images/image_pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/images/image_pull.go b/daemon/images/image_pull.go index 8344ddaf1ea86..154419b3bcd52 100644 --- a/daemon/images/image_pull.go +++ b/daemon/images/image_pull.go @@ -63,7 +63,7 @@ func (i *ImageService) PullImage(ctx context.Context, image, tag string, platfor // we allow the image to have a non-matching architecture. The code // below checks for this situation, and returns a warning to the client, // as well as logging it to the daemon logs. - img, err := i.GetImage(ctx, image, imagetypes.GetImageOpts{Platform: platform}) + img, err := i.GetImage(ctx, ref.String(), imagetypes.GetImageOpts{Platform: platform}) // Note that this is a special case where GetImage returns both an image // and an error: https://github.com/docker/docker/blob/v20.10.7/daemon/images/image.go#L175-L183 From 68b7ba0d03ad355e7adc6025cb9bae2672738ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 18 May 2023 15:28:16 +0200 Subject: [PATCH 020/293] api/inspect: Fix nil RepoTags and RepoDigests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make RepoTags and RepoDigests empty arrays instead of nil. Signed-off-by: Paweł Gronowski (cherry picked from commit 1be26e9f0cb954773616585abb25a1227866b79c) Signed-off-by: Paweł Gronowski --- api/server/router/image/image_routes.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/server/router/image/image_routes.go b/api/server/router/image/image_routes.go index 1500f0e876ffa..a7f8cc9f91ef8 100644 --- a/api/server/router/image/image_routes.go +++ b/api/server/router/image/image_routes.go @@ -282,6 +282,14 @@ func (ir *imageRouter) toImageInspect(img *image.Image) (*types.ImageInspect, er comment = img.History[len(img.History)-1].Comment } + // Make sure we output empty arrays instead of nil. + if repoTags == nil { + repoTags = []string{} + } + if repoDigests == nil { + repoDigests = []string{} + } + return &types.ImageInspect{ ID: img.ID().String(), RepoTags: repoTags, From 4d924c35f7ee7115b1fdf4d6aa4c986ba2a26954 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Wed, 17 May 2023 15:45:59 -0400 Subject: [PATCH 021/293] api/server: allow empty body for POST /commit again The error returned by DecodeConfig was changed in b6d58d749c6d671f0dc19a88b948b772272e145d and caused this to regress. Allow empty request bodies for this endpoint once again. Signed-off-by: Cory Snider (cherry picked from commit 967c7bc5d35107964c5eb5e6bf0d7416558a681b) Signed-off-by: Cory Snider --- api/server/router/container/container_routes.go | 5 ++++- runconfig/config.go | 5 +---- runconfig/errors.go | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/api/server/router/container/container_routes.go b/api/server/router/container/container_routes.go index 891a1a18e916d..5978880124939 100644 --- a/api/server/router/container/container_routes.go +++ b/api/server/router/container/container_routes.go @@ -44,7 +44,7 @@ func (s *containerRouter) postCommit(ctx context.Context, w http.ResponseWriter, } config, _, _, err := s.decoder.DecodeConfig(r.Body) - if err != nil && err != io.EOF { // Do not fail if body is empty. + if err != nil && !errors.Is(err, io.EOF) { // Do not fail if body is empty. return err } @@ -486,6 +486,9 @@ func (s *containerRouter) postContainersCreate(ctx context.Context, w http.Respo config, hostConfig, networkingConfig, err := s.decoder.DecodeConfig(r.Body) if err != nil { + if errors.Is(err, io.EOF) { + return errdefs.InvalidParameter(errors.New("invalid JSON: got EOF while reading request body")) + } return err } version := httputils.VersionFromContext(ctx) diff --git a/runconfig/config.go b/runconfig/config.go index b25d1a8aa3480..3ba1609e9101f 100644 --- a/runconfig/config.go +++ b/runconfig/config.go @@ -77,10 +77,7 @@ func decodeContainerConfig(src io.Reader, si *sysinfo.SysInfo) (*container.Confi func loadJSON(src io.Reader, out interface{}) error { dec := json.NewDecoder(src) if err := dec.Decode(&out); err != nil { - if err == io.EOF { - return validationError("invalid JSON: got EOF while reading request body") - } - return validationError("invalid JSON: " + err.Error()) + return invalidJSONError{Err: err} } if dec.More() { return validationError("unexpected content after JSON") diff --git a/runconfig/errors.go b/runconfig/errors.go index 038fe396609ed..9522a2e0f7a3a 100644 --- a/runconfig/errors.go +++ b/runconfig/errors.go @@ -40,3 +40,17 @@ func (e validationError) Error() string { } func (e validationError) InvalidParameter() {} + +type invalidJSONError struct { + Err error +} + +func (e invalidJSONError) Error() string { + return "invalid JSON: " + e.Err.Error() +} + +func (e invalidJSONError) Unwrap() error { + return e.Err +} + +func (e invalidJSONError) InvalidParameter() {} From d5ad186d49f566eb1acfe289298d235c4b9797ee Mon Sep 17 00:00:00 2001 From: Kevin Alvarez Date: Thu, 18 May 2023 18:24:08 +0200 Subject: [PATCH 022/293] ci(bin-image): distribute build across runners Signed-off-by: CrazyMax (cherry picked from commit 668af4be82a28a728e8f2981e90b7cad224fb2e8) --- .github/workflows/bin-image.yml | 52 ++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index f8992042cf216..9a7e3790327ad 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -24,16 +24,19 @@ jobs: validate-dco: uses: ./.github/workflows/.dco.yml - build: + prepare: runs-on: ubuntu-20.04 - needs: - - validate-dco + outputs: + platforms: ${{ steps.platforms.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v3 - with: - fetch-depth: 0 + - + name: Create platforms matrix + id: platforms + run: | + echo "matrix=$(docker buildx bake bin-image-cross --print | jq -cr '.target."bin-image-cross".platforms')" >>${GITHUB_OUTPUT} - name: Docker meta id: meta @@ -54,6 +57,40 @@ jobs: type=semver,pattern={{version}} type=ref,event=branch type=ref,event=pr + - + name: Rename meta bake definition file + run: | + mv "${{ steps.meta.outputs.bake-file }}" "/tmp/bake-meta.json" + - + name: Upload meta bake definition + uses: actions/upload-artifact@v3 + with: + name: bake-meta + path: /tmp/bake-meta.json + if-no-files-found: error + retention-days: 1 + + build: + runs-on: ubuntu-20.04 + needs: + - validate-dco + - prepare + strategy: + fail-fast: false + matrix: + platform: ${{ fromJson(needs.prepare.outputs.platforms) }} + steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - + name: Download meta bake definition + uses: actions/download-artifact@v3 + with: + name: bake-meta + path: /tmp - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -66,7 +103,8 @@ jobs: with: files: | ./docker-bake.hcl - ${{ steps.meta.outputs.bake-file }} - targets: bin-image-cross + /tmp/bake-meta.json + targets: bin-image set: | + *.platform=${{ matrix.platform }} *.output=type=cacheonly From 07140c0eca5374d6c91bd4b883bd8d09d8524898 Mon Sep 17 00:00:00 2001 From: Kevin Alvarez Date: Thu, 18 May 2023 21:17:22 +0200 Subject: [PATCH 023/293] build: use daemon id as worker id for the graph driver controller Signed-off-by: CrazyMax (cherry picked from commit 6d139e5e950c04325fc91e774ea66dc8b5518288) --- builder/builder-next/builder.go | 1 + builder/builder-next/controller.go | 3 +-- cmd/dockerd/daemon.go | 1 + daemon/daemon.go | 5 +++++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/builder/builder-next/builder.go b/builder/builder-next/builder.go index a262da7fccbb8..f6a89633e6570 100644 --- a/builder/builder-next/builder.go +++ b/builder/builder-next/builder.go @@ -78,6 +78,7 @@ var cacheFields = map[string]bool{ type Opt struct { SessionManager *session.Manager Root string + EngineID string Dist images.DistributionServices ImageTagger mobyexporter.ImageTagger NetworkController *libnetwork.Controller diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index c1a419e3221d8..c84f788bc3ada 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -16,7 +16,6 @@ import ( "github.com/docker/docker/builder/builder-next/adapters/containerimage" "github.com/docker/docker/builder/builder-next/adapters/localinlinecache" "github.com/docker/docker/builder/builder-next/adapters/snapshot" - "github.com/docker/docker/builder/builder-next/exporter" "github.com/docker/docker/builder/builder-next/exporter/mobyexporter" "github.com/docker/docker/builder/builder-next/imagerefchecker" mobyworker "github.com/docker/docker/builder/builder-next/worker" @@ -312,7 +311,7 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt } wopt := mobyworker.Opt{ - ID: exporter.Moby, + ID: opt.EngineID, ContentStore: store, CacheManager: cm, GCPolicy: gcPolicy, diff --git a/cmd/dockerd/daemon.go b/cmd/dockerd/daemon.go index ebd5318520a78..50193d5f97ac2 100644 --- a/cmd/dockerd/daemon.go +++ b/cmd/dockerd/daemon.go @@ -351,6 +351,7 @@ func newRouterOptions(ctx context.Context, config *config.Config, d *daemon.Daem bk, err := buildkit.New(ctx, buildkit.Opt{ SessionManager: sm, Root: filepath.Join(config.Root, "buildkit"), + EngineID: d.ID(), Dist: d.DistributionServices(), ImageTagger: d.ImageService(), NetworkController: d.NetworkController(), diff --git a/daemon/daemon.go b/daemon/daemon.go index 3b9b5244ecffd..a2e375a978a8e 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -131,6 +131,11 @@ type Daemon struct { mdDB *bbolt.DB } +// ID returns the daemon id +func (daemon *Daemon) ID() string { + return daemon.id +} + // StoreHosts stores the addresses the daemon is listening on func (daemon *Daemon) StoreHosts(hosts []string) { if daemon.hosts == nil { From f9b886c01b36e605ff822f18e9f924cae94ae415 Mon Sep 17 00:00:00 2001 From: Nolan Miles Date: Wed, 17 May 2023 20:49:40 -0400 Subject: [PATCH 024/293] add mirror to daemon reload test for insecure registries Signed-off-by: Nolan Miles (cherry picked from commit 3b15156e4d317b093ddcbb59e0ce75cadc6b97dc) Signed-off-by: Sebastiaan van Stijn --- daemon/reload_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/daemon/reload_test.go b/daemon/reload_test.go index 0d72f1dd4c93d..5e35bbad47b34 100644 --- a/daemon/reload_test.go +++ b/daemon/reload_test.go @@ -238,13 +238,19 @@ func TestDaemonReloadInsecureRegistries(t *testing.T) { "docker3.example.com", // this will be newly added } + mirrors := []string{ + "https://mirror.test.example.com", + } + valuesSets := make(map[string]interface{}) valuesSets["insecure-registries"] = insecureRegistries + valuesSets["registry-mirrors"] = mirrors newConfig := &config.Config{ CommonConfig: config.CommonConfig{ ServiceOptions: registry.ServiceOptions{ InsecureRegistries: insecureRegistries, + Mirrors: mirrors, }, ValuesSet: valuesSets, }, From 3467ba6451e38c449aa8bd1c1c8f186e7f6f6ead Mon Sep 17 00:00:00 2001 From: Nolan Miles Date: Wed, 17 May 2023 20:50:12 -0400 Subject: [PATCH 025/293] reorder load funcs to match newServiceConfig()'s order Signed-off-by: Nolan Miles (cherry picked from commit f3645a2aa32d4c42ce44f84a7425c3f0278177d2) Signed-off-by: Sebastiaan van Stijn --- daemon/reload.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/reload.go b/daemon/reload.go index 20f1b8eacf5bb..a5bef627126db 100644 --- a/daemon/reload.go +++ b/daemon/reload.go @@ -63,10 +63,10 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) { if err := daemon.reloadAllowNondistributableArtifacts(conf, attributes); err != nil { return err } - if err := daemon.reloadInsecureRegistries(conf, attributes); err != nil { + if err := daemon.reloadRegistryMirrors(conf, attributes); err != nil { return err } - if err := daemon.reloadRegistryMirrors(conf, attributes); err != nil { + if err := daemon.reloadInsecureRegistries(conf, attributes); err != nil { return err } if err := daemon.reloadLiveRestore(conf, attributes); err != nil { From 0869b089e4b1633ac8c9c805b9293030a8782664 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Thu, 18 May 2023 14:10:44 -0400 Subject: [PATCH 026/293] libnetwork: just forward the external DNS response Our resolver is just a forwarder for external DNS so it should act like it. Unless it's a server failure or refusal, take the response at face value and forward it along to the client. RFC 8020 is only applicable to caching recursive name servers and our resolver is neither caching nor recursive. Signed-off-by: Cory Snider (cherry picked from commit 41356227f24801510f9eb173ffe4ddd4768587db) Signed-off-by: Sebastiaan van Stijn --- libnetwork/resolver.go | 27 ++--------------- libnetwork/resolver_test.go | 58 +++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 24 deletions(-) diff --git a/libnetwork/resolver.go b/libnetwork/resolver.go index e8e694403953c..1304e95df2a30 100644 --- a/libnetwork/resolver.go +++ b/libnetwork/resolver.go @@ -449,7 +449,6 @@ func (r *Resolver) dialExtDNS(proto string, server extDNSEntry) (net.Conn, error } func (r *Resolver) forwardExtDNS(proto string, query *dns.Msg) *dns.Msg { - queryName, queryType := query.Question[0].Name, query.Question[0].Qtype for _, extDNS := range r.extDNSList { if extDNS.IPStr == "" { break @@ -477,20 +476,7 @@ func (r *Resolver) forwardExtDNS(proto string, query *dns.Msg) *dns.Msg { case dns.RcodeServerFailure, dns.RcodeRefused: // Server returned FAILURE: continue with the next external DNS server // Server returned REFUSED: this can be a transitional status, so continue with the next external DNS server - logrus.Debugf("[resolver] external DNS %s:%s responded with %s for %q", proto, extDNS.IPStr, statusString(resp.Rcode), queryName) - continue - case dns.RcodeNameError: - // Server returned NXDOMAIN. Stop resolution if it's an authoritative answer (see RFC 8020: https://tools.ietf.org/html/rfc8020#section-2) - logrus.Debugf("[resolver] external DNS %s:%s responded with %s for %q", proto, extDNS.IPStr, statusString(resp.Rcode), queryName) - if resp.Authoritative { - break - } - continue - case dns.RcodeSuccess: - // All is well - default: - // Server gave some error. Log the error, and continue with the next external DNS server - logrus.Debugf("[resolver] external DNS %s:%s responded with %s (code %d) for %q", proto, extDNS.IPStr, statusString(resp.Rcode), resp.Rcode, queryName) + logrus.Debugf("[resolver] external DNS %s:%s returned failure:\n%s", proto, extDNS.IPStr, resp) continue } answers := 0 @@ -509,8 +495,8 @@ func (r *Resolver) forwardExtDNS(proto string, query *dns.Msg) *dns.Msg { r.backend.HandleQueryResp(h.Name, ip) } } - if resp.Answer == nil || answers == 0 { - logrus.Debugf("[resolver] external DNS %s:%s did not return any %s records for %q", proto, extDNS.IPStr, dns.TypeToString[queryType], queryName) + if len(resp.Answer) == 0 { + logrus.Debugf("[resolver] external DNS %s:%s returned response with no answers:\n%s", proto, extDNS.IPStr, resp) } resp.Compress = true return resp @@ -558,10 +544,3 @@ func (r *Resolver) exchange(proto string, extDNS extDNSEntry, query *dns.Msg) *d } return resp } - -func statusString(responseCode int) string { - if s, ok := dns.RcodeToString[responseCode]; ok { - return s - } - return "UNKNOWN" -} diff --git a/libnetwork/resolver_test.go b/libnetwork/resolver_test.go index e782de6e2a3c9..4637a4298ebbd 100644 --- a/libnetwork/resolver_test.go +++ b/libnetwork/resolver_test.go @@ -13,6 +13,7 @@ import ( "github.com/miekg/dns" "github.com/sirupsen/logrus" "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" "gotest.tools/v3/skip" ) @@ -457,3 +458,60 @@ type badSRVDNSBackend struct{ noopDNSBackend } func (badSRVDNSBackend) ResolveService(name string) ([]*net.SRV, []net.IP) { return []*net.SRV{nil, nil, nil}, nil // Mismatched slice lengths } + +func TestProxyNXDOMAIN(t *testing.T) { + mockSOA, err := dns.NewRR(". 86367 IN SOA a.root-servers.net. nstld.verisign-grs.com. 2023051800 1800 900 604800 86400\n") + assert.NilError(t, err) + assert.Assert(t, mockSOA != nil) + + serveStarted := make(chan struct{}) + srv := &dns.Server{ + Net: "udp", + Addr: "127.0.0.1:0", + Handler: dns.HandlerFunc(func(w dns.ResponseWriter, r *dns.Msg) { + msg := new(dns.Msg).SetRcode(r, dns.RcodeNameError) + msg.Ns = append(msg.Ns, dns.Copy(mockSOA)) + w.WriteMsg(msg) + }), + NotifyStartedFunc: func() { close(serveStarted) }, + } + serveDone := make(chan error, 1) + go func() { + defer close(serveDone) + serveDone <- srv.ListenAndServe() + }() + + select { + case err := <-serveDone: + t.Fatal(err) + case <-serveStarted: + } + + defer func() { + if err := srv.Shutdown(); err != nil { + t.Error(err) + } + <-serveDone + }() + + srvAddr := srv.PacketConn.LocalAddr().(*net.UDPAddr) + rsv := NewResolver("", true, noopDNSBackend{}) + rsv.SetExtServers([]extDNSEntry{ + {IPStr: srvAddr.IP.String(), port: uint16(srvAddr.Port), HostLoopback: true}, + }) + + // The resolver logs lots of valuable info at level debug. Redirect it + // to t.Log() so the log spew is emitted only if the test fails. + defer redirectLogrusTo(t)() + + w := &tstwriter{localAddr: srv.PacketConn.LocalAddr()} + q := new(dns.Msg).SetQuestion("example.net.", dns.TypeA) + rsv.serveDNS(w, q) + resp := w.GetResponse() + checkNonNullResponse(t, resp) + t.Log("Response:\n" + resp.String()) + checkDNSResponseCode(t, resp, dns.RcodeNameError) + assert.Assert(t, is.Len(resp.Answer, 0)) + assert.Assert(t, is.Len(resp.Ns, 1)) + assert.Equal(t, resp.Ns[0].String(), mockSOA.String()) +} From 47a3dad256e9ec6f4a2494cabc189ae709265d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 19 May 2023 18:11:25 +0200 Subject: [PATCH 027/293] c8d/list: Show layerless images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 34964c2454e32295866a1920a4d50982d10f7ab4) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_list.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/daemon/containerd/image_list.go b/daemon/containerd/image_list.go index 72cca0d18aa3e..9128f88797d7c 100644 --- a/daemon/containerd/image_list.go +++ b/daemon/containerd/image_list.go @@ -525,9 +525,12 @@ func getManifestPlatform(ctx context.Context, store content.Provider, manifestDe return platforms.Normalize(platform), nil } -// isImageManifest returns true if the manifest has any layer that is a known image layer. +// isImageManifest returns true if the manifest has no layers or any of its layers is a known image layer. // Some manifests use the image media type for compatibility, even if they are not a real image. func isImageManifest(mfst v1.Manifest) bool { + if len(mfst.Layers) == 0 { + return true + } for _, l := range mfst.Layers { if images.IsLayerType(l.MediaType) { return true From 876f5eda51e435f6f4201760304628da5832e645 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Fri, 19 May 2023 09:03:33 -0400 Subject: [PATCH 028/293] libnetwork: make resolver tests less confusing tstwriter mocks the server-side connection between the resolver and the container, not the resolver and the external DNS server, so returning the external DNS server's address as w.LocalAddr() is technically incorrect and misleading. Only the protocols need to match as the resolver uses the client's choice of protocol to determine which protocol to use when forwarding the query to the external DNS server. While this change has no material impact on the tests, it makes the tests slightly more comprehensible for the next person. Signed-off-by: Cory Snider (cherry picked from commit 0cc6e445d7137716de09035a6735338db945efe5) Signed-off-by: Cory Snider --- libnetwork/resolver_test.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/libnetwork/resolver_test.go b/libnetwork/resolver_test.go index 4637a4298ebbd..dc6f4ffd70be1 100644 --- a/libnetwork/resolver_test.go +++ b/libnetwork/resolver_test.go @@ -19,16 +19,22 @@ import ( // a simple/null address type that will be used to fake a local address for unit testing type tstaddr struct { + network string } -func (a *tstaddr) Network() string { return "tcp" } +func (a *tstaddr) Network() string { + if a.network != "" { + return a.network + } + return "tcp" +} -func (a *tstaddr) String() string { return "127.0.0.1" } +func (a *tstaddr) String() string { return "(fake)" } // a simple writer that implements dns.ResponseWriter for unit testing purposes type tstwriter struct { - localAddr net.Addr - msg *dns.Msg + network string + msg *dns.Msg } func (w *tstwriter) WriteMsg(m *dns.Msg) (err error) { @@ -39,13 +45,12 @@ func (w *tstwriter) WriteMsg(m *dns.Msg) (err error) { func (w *tstwriter) Write(m []byte) (int, error) { return 0, nil } func (w *tstwriter) LocalAddr() net.Addr { - if w.localAddr != nil { - return w.localAddr - } - return new(tstaddr) + return &tstaddr{network: w.network} } -func (w *tstwriter) RemoteAddr() net.Addr { return new(tstaddr) } +func (w *tstwriter) RemoteAddr() net.Addr { + return &tstaddr{network: w.network} +} func (w *tstwriter) TsigStatus() error { return nil } @@ -380,7 +385,7 @@ func TestOversizedDNSReply(t *testing.T) { // to t.Log() so the log spew is emitted only if the test fails. defer redirectLogrusTo(t)() - w := &tstwriter{localAddr: srv.LocalAddr()} + w := &tstwriter{network: srvAddr.Network()} q := new(dns.Msg).SetQuestion("s3.amazonaws.com.", dns.TypeA) rsv.serveDNS(w, q) resp := w.GetResponse() @@ -504,7 +509,7 @@ func TestProxyNXDOMAIN(t *testing.T) { // to t.Log() so the log spew is emitted only if the test fails. defer redirectLogrusTo(t)() - w := &tstwriter{localAddr: srv.PacketConn.LocalAddr()} + w := &tstwriter{network: srvAddr.Network()} q := new(dns.Msg).SetQuestion("example.net.", dns.TypeA) rsv.serveDNS(w, q) resp := w.GetResponse() From ef1545ed4a8d7e40d74a1c4b04338de32d76d23d Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Fri, 19 May 2023 11:27:15 -0400 Subject: [PATCH 029/293] libnetwork: leave global logger alone in tests Swapping out the global logger on the fly is causing tests to flake out by logging to a test's log output after the test function has returned. Refactor Resolver to use a dependency-injected logger and the resolver unit tests to inject a private logger instance into the Resolver under test. Signed-off-by: Cory Snider (cherry picked from commit d4f3858a405e6761edc8281970fb97f623ef3df8) Signed-off-by: Cory Snider --- libnetwork/resolver.go | 40 ++++++++++++++++++++++--------------- libnetwork/resolver_test.go | 25 ++++++++++------------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/libnetwork/resolver.go b/libnetwork/resolver.go index 1304e95df2a30..ab19b7b08fc0b 100644 --- a/libnetwork/resolver.go +++ b/libnetwork/resolver.go @@ -71,6 +71,7 @@ type Resolver struct { listenAddress string proxyDNS bool startCh chan struct{} + logger *logrus.Logger fwdSem *semaphore.Weighted // Limit the number of concurrent external DNS requests in-flight logInverval rate.Sometimes // Rate-limit logging about hitting the fwdSem limit @@ -89,6 +90,13 @@ func NewResolver(address string, proxyDNS bool, backend DNSBackend) *Resolver { } } +func (r *Resolver) log() *logrus.Logger { + if r.logger == nil { + return logrus.StandardLogger() + } + return r.logger +} + // SetupFunc returns the setup function that should be run in the container's // network namespace. func (r *Resolver) SetupFunc(port int) func() { @@ -140,7 +148,7 @@ func (r *Resolver) Start() error { r.server = s go func() { if err := s.ActivateAndServe(); err != nil { - logrus.WithError(err).Error("[resolver] failed to start PacketConn DNS server") + r.log().WithError(err).Error("[resolver] failed to start PacketConn DNS server") } }() @@ -148,7 +156,7 @@ func (r *Resolver) Start() error { r.tcpServer = tcpServer go func() { if err := tcpServer.ActivateAndServe(); err != nil { - logrus.WithError(err).Error("[resolver] failed to start TCP DNS server") + r.log().WithError(err).Error("[resolver] failed to start TCP DNS server") } }() return nil @@ -249,7 +257,7 @@ func (r *Resolver) handleIPQuery(query *dns.Msg, ipType int) (*dns.Msg, error) { if addr == nil && ipv6Miss { // Send a reply without any Answer sections - logrus.Debugf("[resolver] lookup name %s present without IPv6 address", name) + r.log().Debugf("[resolver] lookup name %s present without IPv6 address", name) resp := createRespMsg(query) return resp, nil } @@ -257,7 +265,7 @@ func (r *Resolver) handleIPQuery(query *dns.Msg, ipType int) (*dns.Msg, error) { return nil, nil } - logrus.Debugf("[resolver] lookup for %s: IP %v", name, addr) + r.log().Debugf("[resolver] lookup for %s: IP %v", name, addr) resp := createRespMsg(query) if len(addr) > 1 { @@ -298,7 +306,7 @@ func (r *Resolver) handlePTRQuery(query *dns.Msg) (*dns.Msg, error) { return nil, nil } - logrus.Debugf("[resolver] lookup for IP %s: name %s", name, host) + r.log().Debugf("[resolver] lookup for IP %s: name %s", name, host) fqdn := dns.Fqdn(host) resp := new(dns.Msg) @@ -365,17 +373,17 @@ func (r *Resolver) serveDNS(w dns.ResponseWriter, query *dns.Msg) { case dns.TypeSRV: resp, err = r.handleSRVQuery(query) default: - logrus.Debugf("[resolver] query type %s is not supported by the embedded DNS and will be forwarded to external DNS", dns.TypeToString[queryType]) + r.log().Debugf("[resolver] query type %s is not supported by the embedded DNS and will be forwarded to external DNS", dns.TypeToString[queryType]) } reply := func(msg *dns.Msg) { if err = w.WriteMsg(msg); err != nil { - logrus.WithError(err).Errorf("[resolver] failed to write response") + r.log().WithError(err).Errorf("[resolver] failed to write response") } } if err != nil { - logrus.WithError(err).Errorf("[resolver] failed to handle query: %s (%s)", queryName, dns.TypeToString[queryType]) + r.log().WithError(err).Errorf("[resolver] failed to handle query: %s (%s)", queryName, dns.TypeToString[queryType]) reply(new(dns.Msg).SetRcode(query, dns.RcodeServerFailure)) return } @@ -460,7 +468,7 @@ func (r *Resolver) forwardExtDNS(proto string, query *dns.Msg) *dns.Msg { cancel() if err != nil { r.logInverval.Do(func() { - logrus.Errorf("[resolver] more than %v concurrent queries", maxConcurrent) + r.log().Errorf("[resolver] more than %v concurrent queries", maxConcurrent) }) return new(dns.Msg).SetRcode(query, dns.RcodeRefused) } @@ -476,7 +484,7 @@ func (r *Resolver) forwardExtDNS(proto string, query *dns.Msg) *dns.Msg { case dns.RcodeServerFailure, dns.RcodeRefused: // Server returned FAILURE: continue with the next external DNS server // Server returned REFUSED: this can be a transitional status, so continue with the next external DNS server - logrus.Debugf("[resolver] external DNS %s:%s returned failure:\n%s", proto, extDNS.IPStr, resp) + r.log().Debugf("[resolver] external DNS %s:%s returned failure:\n%s", proto, extDNS.IPStr, resp) continue } answers := 0 @@ -486,17 +494,17 @@ func (r *Resolver) forwardExtDNS(proto string, query *dns.Msg) *dns.Msg { case dns.TypeA: answers++ ip := rr.(*dns.A).A - logrus.Debugf("[resolver] received A record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr) + r.log().Debugf("[resolver] received A record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr) r.backend.HandleQueryResp(h.Name, ip) case dns.TypeAAAA: answers++ ip := rr.(*dns.AAAA).AAAA - logrus.Debugf("[resolver] received AAAA record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr) + r.log().Debugf("[resolver] received AAAA record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr) r.backend.HandleQueryResp(h.Name, ip) } } if len(resp.Answer) == 0 { - logrus.Debugf("[resolver] external DNS %s:%s returned response with no answers:\n%s", proto, extDNS.IPStr, resp) + r.log().Debugf("[resolver] external DNS %s:%s returned response with no answers:\n%s", proto, extDNS.IPStr, resp) } resp.Compress = true return resp @@ -508,12 +516,12 @@ func (r *Resolver) forwardExtDNS(proto string, query *dns.Msg) *dns.Msg { func (r *Resolver) exchange(proto string, extDNS extDNSEntry, query *dns.Msg) *dns.Msg { extConn, err := r.dialExtDNS(proto, extDNS) if err != nil { - logrus.WithError(err).Warn("[resolver] connect failed") + r.log().WithError(err).Warn("[resolver] connect failed") return nil } defer extConn.Close() - log := logrus.WithFields(logrus.Fields{ + log := r.log().WithFields(logrus.Fields{ "dns-server": extConn.RemoteAddr().Network() + ":" + extConn.RemoteAddr().String(), "client-addr": extConn.LocalAddr().Network() + ":" + extConn.LocalAddr().String(), "question": query.Question[0].String(), @@ -534,7 +542,7 @@ func (r *Resolver) exchange(proto string, extDNS extDNSEntry, query *dns.Msg) *d UDPSize: dns.MaxMsgSize, }).ExchangeWithConn(query, &dns.Conn{Conn: extConn}) if err != nil { - logrus.WithError(err).Errorf("[resolver] failed to query DNS server: %s, query: %s", extConn.RemoteAddr().String(), query.Question[0].String()) + r.log().WithError(err).Errorf("[resolver] failed to query DNS server: %s, query: %s", extConn.RemoteAddr().String(), query.Question[0].String()) return nil } diff --git a/libnetwork/resolver_test.go b/libnetwork/resolver_test.go index dc6f4ffd70be1..103aa98755e02 100644 --- a/libnetwork/resolver_test.go +++ b/libnetwork/resolver_test.go @@ -377,14 +377,13 @@ func TestOversizedDNSReply(t *testing.T) { srvAddr := srv.LocalAddr().(*net.UDPAddr) rsv := NewResolver("", true, noopDNSBackend{}) + // The resolver logs lots of valuable info at level debug. Redirect it + // to t.Log() so the log spew is emitted only if the test fails. + rsv.logger = testLogger(t) rsv.SetExtServers([]extDNSEntry{ {IPStr: srvAddr.IP.String(), port: uint16(srvAddr.Port), HostLoopback: true}, }) - // The resolver logs lots of valuable info at level debug. Redirect it - // to t.Log() so the log spew is emitted only if the test fails. - defer redirectLogrusTo(t)() - w := &tstwriter{network: srvAddr.Network()} q := new(dns.Msg).SetQuestion("s3.amazonaws.com.", dns.TypeA) rsv.serveDNS(w, q) @@ -396,14 +395,11 @@ func TestOversizedDNSReply(t *testing.T) { checkDNSRRType(t, resp.Answer[0].Header().Rrtype, dns.TypeA) } -func redirectLogrusTo(t *testing.T) func() { - oldLevel, oldOut := logrus.StandardLogger().Level, logrus.StandardLogger().Out - logrus.StandardLogger().SetLevel(logrus.DebugLevel) - logrus.SetOutput(tlogWriter{t}) - return func() { - logrus.StandardLogger().SetLevel(oldLevel) - logrus.StandardLogger().SetOutput(oldOut) - } +func testLogger(t *testing.T) *logrus.Logger { + logger := logrus.New() + logger.SetLevel(logrus.DebugLevel) + logger.SetOutput(tlogWriter{t}) + return logger } type tlogWriter struct{ t *testing.T } @@ -445,9 +441,8 @@ func TestReplySERVFAIL(t *testing.T) { } for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - defer redirectLogrusTo(t) - rsv := NewResolver("", tt.proxyDNS, badSRVDNSBackend{}) + rsv.logger = testLogger(t) w := &tstwriter{} rsv.serveDNS(w, tt.q) resp := w.GetResponse() @@ -507,7 +502,7 @@ func TestProxyNXDOMAIN(t *testing.T) { // The resolver logs lots of valuable info at level debug. Redirect it // to t.Log() so the log spew is emitted only if the test fails. - defer redirectLogrusTo(t)() + rsv.logger = testLogger(t) w := &tstwriter{network: srvAddr.Network()} q := new(dns.Msg).SetQuestion("example.net.", dns.TypeA) From 992dc33fc5aabda31d55cb2758e56e6b8fd280b7 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Fri, 19 May 2023 18:16:00 -0400 Subject: [PATCH 030/293] libnetwork/osl: restore the right thread's netns osl.setIPv6 mistakenly captured the calling goroutine's thread's network namespace instead of the network namespace of the thread getting its namespace temporarily changed. As this function appears to only be called from contexts in the process's initial network namespace, this mistake would be of little consequence at runtime. The libnetwork unit tests, on the other hand, unshare network namespaces so as not to interfere with each other or the host's network namespace. But due to this bug, the isolation backfires and the network namespace of goroutines used by a test which are expected to be in the initial network namespace can randomly become the isolated network namespace of some other test. Symptoms include a loopback network server running in one goroutine being inexplicably and randomly being unreachable by a client in another goroutine. Capture the original network namespace of the thread from the thread to be tampered with, after locking the goroutine to the thread. Signed-off-by: Cory Snider (cherry picked from commit 6d798641351a18390f80570bf1ef554c54cfb71b) Signed-off-by: Cory Snider --- libnetwork/osl/namespace_linux.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/libnetwork/osl/namespace_linux.go b/libnetwork/osl/namespace_linux.go index 6da6f95b01826..9f22d8077208b 100644 --- a/libnetwork/osl/namespace_linux.go +++ b/libnetwork/osl/namespace_linux.go @@ -600,24 +600,29 @@ func (n *networkNamespace) checkLoV6() { } func setIPv6(nspath, iface string, enable bool) error { - origNS, err := netns.Get() - if err != nil { - return fmt.Errorf("failed to get current network namespace: %w", err) - } - defer origNS.Close() - - namespace, err := netns.GetFromPath(nspath) - if err != nil { - return fmt.Errorf("failed get network namespace %q: %w", nspath, err) - } - defer namespace.Close() - errCh := make(chan error, 1) go func() { defer close(errCh) + namespace, err := netns.GetFromPath(nspath) + if err != nil { + errCh <- fmt.Errorf("failed get network namespace %q: %w", nspath, err) + return + } + defer namespace.Close() + runtime.LockOSThread() + + origNS, err := netns.Get() + if err != nil { + runtime.UnlockOSThread() + errCh <- fmt.Errorf("failed to get current network namespace: %w", err) + return + } + defer origNS.Close() + if err = netns.Set(namespace); err != nil { + runtime.UnlockOSThread() errCh <- fmt.Errorf("setting into container netns %q failed: %w", nspath, err) return } From baf1fd1c3fe57265b1e04d4e262124208f3b6fb4 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Fri, 19 May 2023 19:27:29 -0400 Subject: [PATCH 031/293] libnetwork: check for netns leaks from prior tests TestProxyNXDOMAIN has proven to be susceptible to failing as a consequence of unlocked threads being set to the wrong network namespace. As the failure mode looks a lot like a bug in the test itself, it seems prudent to add a check for mismatched namespaces to the test so we will know for next time that the root cause lies elsewhere. Signed-off-by: Cory Snider (cherry picked from commit 871cf72363cfa402c2c5932658f797e7112a4796) Signed-off-by: Cory Snider --- libnetwork/resolver_test.go | 7 ++++ libnetwork/testutils/sanity_linux.go | 43 +++++++++++++++++++++++++ libnetwork/testutils/sanity_notlinux.go | 11 +++++++ 3 files changed, 61 insertions(+) create mode 100644 libnetwork/testutils/sanity_linux.go create mode 100644 libnetwork/testutils/sanity_notlinux.go diff --git a/libnetwork/resolver_test.go b/libnetwork/resolver_test.go index 103aa98755e02..733e1992ded07 100644 --- a/libnetwork/resolver_test.go +++ b/libnetwork/resolver_test.go @@ -494,6 +494,13 @@ func TestProxyNXDOMAIN(t *testing.T) { <-serveDone }() + // This test, by virtue of running a server and client in different + // not-locked-to-thread goroutines, happens to be a good canary for + // whether we are leaking unlocked OS threads set to the wrong network + // namespace. Make a best-effort attempt to detect that situation so we + // are not left chasing ghosts next time. + testutils.AssertSocketSameNetNS(t, srv.PacketConn.(*net.UDPConn)) + srvAddr := srv.PacketConn.LocalAddr().(*net.UDPAddr) rsv := NewResolver("", true, noopDNSBackend{}) rsv.SetExtServers([]extDNSEntry{ diff --git a/libnetwork/testutils/sanity_linux.go b/libnetwork/testutils/sanity_linux.go new file mode 100644 index 0000000000000..8c85e1a8968f6 --- /dev/null +++ b/libnetwork/testutils/sanity_linux.go @@ -0,0 +1,43 @@ +package testutils + +import ( + "errors" + "syscall" + "testing" + + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" + "gotest.tools/v3/assert" +) + +// AssertSocketSameNetNS makes a best-effort attempt to assert that conn is in +// the same network namespace as the current goroutine's thread. +func AssertSocketSameNetNS(t testing.TB, conn syscall.Conn) { + t.Helper() + + sc, err := conn.SyscallConn() + assert.NilError(t, err) + sc.Control(func(fd uintptr) { + srvnsfd, err := unix.IoctlRetInt(int(fd), unix.SIOCGSKNS) + if err != nil { + if errors.Is(err, unix.EPERM) { + t.Log("Cannot determine socket's network namespace. Do we have CAP_NET_ADMIN?") + return + } + if errors.Is(err, unix.ENOSYS) { + t.Log("Cannot query socket's network namespace due to missing kernel support.") + return + } + t.Fatal(err) + } + srvns := netns.NsHandle(srvnsfd) + defer srvns.Close() + + curns, err := netns.Get() + assert.NilError(t, err) + defer curns.Close() + if !srvns.Equal(curns) { + t.Fatalf("Socket is in network namespace %s, but test goroutine is in %s", srvns, curns) + } + }) +} diff --git a/libnetwork/testutils/sanity_notlinux.go b/libnetwork/testutils/sanity_notlinux.go new file mode 100644 index 0000000000000..ed58a6dbdaaf7 --- /dev/null +++ b/libnetwork/testutils/sanity_notlinux.go @@ -0,0 +1,11 @@ +//go:build !linux + +package testutils + +import ( + "syscall" + "testing" +) + +// AssertSocketSameNetNS is a no-op on platforms other than Linux. +func AssertSocketSameNetNS(t testing.TB, conn syscall.Conn) {} From 5276c2b6e090c11baceec8dfd138d74523c40587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 24 May 2023 11:16:13 +0200 Subject: [PATCH 032/293] c8d/pull: Use same progress action as distribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker with containerd integration emits "Exists" progress action when a layer of the currently pulled image already exists. This is different from the non-c8d Docker which emits "Already exists". This makes both implementations consistent by emitting backwards compatible "Already exists" action. Signed-off-by: Paweł Gronowski (cherry picked from commit a7bc65fbd83b367bf827d8fbbef220b2a8e6f406) Signed-off-by: Paweł Gronowski --- daemon/containerd/progress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/containerd/progress.go b/daemon/containerd/progress.go index 31bce71a04ad5..80ea23b6aca01 100644 --- a/daemon/containerd/progress.go +++ b/daemon/containerd/progress.go @@ -152,7 +152,7 @@ func (p pullProgress) UpdateProgress(ctx context.Context, ongoing *jobs, out pro } else if p.ShowExists { out.WriteProgress(progress.Progress{ ID: stringid.TruncateID(j.Digest.Encoded()), - Action: "Exists", + Action: "Already exists", HideCounts: true, LastUpdate: true, }) From a753ca64e23d3e9c7a19f0c60c5f3c8163dc8f1d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 24 May 2023 08:51:05 -0600 Subject: [PATCH 033/293] hack/make/.binary: don't use "netgo" when building Windows binaries Starting with go1.19, the Go runtime on Windows now supports the `netgo` build- flag to use a native Go DNS resolver. Prior to that version, the build-flag only had an effect on non-Windows platforms. When using the `netgo` build-flag, the Windows's host resolver is not used, and as a result, custom entries in `etc/hosts` are ignored, which is a change in behavior from binaries compiled with older versions of the Go runtime. From the go1.19 release notes: https://go.dev/doc/go1.19#net > Resolver.PreferGo is now implemented on Windows and Plan 9. It previously > only worked on Unix platforms. Combined with Dialer.Resolver and Resolver.Dial, > it's now possible to write portable programs and be in control of all DNS name > lookups when dialing. > > The net package now has initial support for the netgo build tag on Windows. > When used, the package uses the Go DNS client (as used by Resolver.PreferGo) > instead of asking Windows for DNS results. The upstream DNS server it discovers > from Windows may not yet be correct with complex system network configurations, > however. Our Windows binaries are compiled with the "static" (`make/binary-daemon`) script, which has the `netgo` option set by default. This patch unsets the `netgo` option when cross-compiling for Windows. Co-authored-by: Bjorn Neergaard Signed-off-by: Bjorn Neergaard (cherry picked from commit 53d1b12bc014b4243e9439fc2610eb4ef863659f) Signed-off-by: Bjorn Neergaard --- hack/make/.binary | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hack/make/.binary b/hack/make/.binary index 39c00cd50c943..38124682090ea 100644 --- a/hack/make/.binary +++ b/hack/make/.binary @@ -51,6 +51,18 @@ source "${MAKEDIR}/.go-autogen" fi fi + # XXX: Disable netgo on Windows and use Window's system resolver instead. + # + # go1.19 and newer added support for netgo on Windows (https://go.dev/doc/go1.19#net), + # which won't ask Windows for DNS results, and hence may be ignoring + # custom "C:\Windows\System32\drivers\etc\hosts". + # See https://github.com/moby/moby/issues/45251#issuecomment-1561001817 + # https://github.com/moby/moby/issues/45251, and + # https://go-review.googlesource.com/c/go/+/467335 + if [ "$(go env GOOS)" = "windows" ]; then + BUILDFLAGS=("${BUILDFLAGS[@]/netgo/}") + fi + # only necessary for non-sandboxed invocation where TARGETPLATFORM is empty PLATFORM_NAME=$TARGETPLATFORM if [ -z "$PLATFORM_NAME" ]; then From 8a4b7c5af85557db80527477fe242382705dc929 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 13 Apr 2023 12:16:27 +0200 Subject: [PATCH 034/293] Add testenv.UsingSnapshotter utility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To allow skipping integration tests that don't apply to the containerd snapshotter. Signed-off-by: Sebastiaan van Stijn Signed-off-by: Paweł Gronowski (cherry picked from commit 43735478577eff35687434d5134bc5d536e6647e) Signed-off-by: Paweł Gronowski --- testutil/environment/environment.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/testutil/environment/environment.go b/testutil/environment/environment.go index 7007008fec489..fda7cf1265a2a 100644 --- a/testutil/environment/environment.go +++ b/testutil/environment/environment.go @@ -193,6 +193,13 @@ func (e *Execution) IsUserNamespaceInKernel() bool { return true } +// UsingSnapshotter returns whether containerd snapshotters are used for the +// tests by checking if the "TEST_INTEGRATION_USE_SNAPSHOTTER" is set to a +// non-empty value. +func (e *Execution) UsingSnapshotter() bool { + return os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" +} + // HasExistingImage checks whether there is an image with the given reference. // Note that this is done by filtering and then checking whether there were any // results -- so ambiguous references might result in false-positives. From 9ee7d30aefb15f800742e354942064d2b63c240e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 22 May 2023 11:47:28 +0200 Subject: [PATCH 035/293] hack/ensure-emptyfs: Create dangling image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 3a31f81838ba80d533e6156c7e571a56a575e4e0) Signed-off-by: Paweł Gronowski --- hack/make/.ensure-emptyfs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/hack/make/.ensure-emptyfs b/hack/make/.ensure-emptyfs index db15aabd533a9..db14275850ca2 100644 --- a/hack/make/.ensure-emptyfs +++ b/hack/make/.ensure-emptyfs @@ -1,7 +1,12 @@ #!/usr/bin/env bash set -e -if ! docker image inspect emptyfs > /dev/null; then +function imageNotPresent { + local img="$1" + ! docker image inspect "$img" > /dev/null 2> /dev/null +} + +if imageNotPresent "emptyfs"; then # build a "docker save" tarball for "emptyfs" # see https://github.com/docker/docker/pull/5262 # and also https://github.com/docker/docker/issues/4242 @@ -24,3 +29,27 @@ if ! docker image inspect emptyfs > /dev/null; then ) rm -rf "$dir" fi + +# without c8d image store, image id is the config's id +dangling_cfg=0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43 +# with c8d image store, image id is the id of manifest/manifest list. +dangling_mfst=16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456 +if imageNotPresent "$dangling_cfg" && imageNotPresent "$dangling_mfst"; then + dir="$DEST/dangling" + mkdir -p "$dir" + ( + cd "$dir" + printf '{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456","size":264,"annotations":{"org.opencontainers.image.created":"2023-05-19T08:00:44Z"},"platform":{"architecture":"amd64","os":"linux"}}]}' > index.json + printf '[{"Config":"blobs/sha256/0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43","RepoTags":null,"Layers":null}]' > manifest.json + mkdir -p blobs/sha256 + printf '{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43","size":390},"layers":[]}' > blobs/sha256/$dangling_mfst + printf '{"architecture":"amd64","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"WorkingDir":"/","Labels":{"org.mobyproject.test.specialimage":"1"},"OnBuild":null},"created":null,"history":[{"created_by":"LABEL org.mobyproject.test.specialimage=1","comment":"buildkit.dockerfile.v0","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":null}}' > blobs/sha256/$dangling_cfg + tar -cf layer.tar --files-from /dev/null + ) + ( + [ -n "$TESTDEBUG" ] && set -x + tar -cC "$dir" . | docker load + ) + rm -rf "$dir" + +fi From 27df42255c7d8a176629296e18f2bb69d0624a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 22 May 2023 11:53:14 +0200 Subject: [PATCH 036/293] hack: Rename .ensure-emptyfs to .build-empty-images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit a93aadc2e60ee8b824a26391ff82c35cde2b0cda) Signed-off-by: Paweł Gronowski --- Dockerfile.e2e | 4 ++-- hack/make/{.ensure-emptyfs => .build-empty-images} | 0 hack/make/.integration-daemon-setup | 2 +- hack/test/e2e-run.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename hack/make/{.ensure-emptyfs => .build-empty-images} (100%) diff --git a/Dockerfile.e2e b/Dockerfile.e2e index 9dc49e845a05c..2b41a7e65e4de 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -71,8 +71,8 @@ RUN apk --no-cache add \ tar \ xz -COPY hack/test/e2e-run.sh /scripts/run.sh -COPY hack/make/.ensure-emptyfs /scripts/ensure-emptyfs.sh +COPY hack/test/e2e-run.sh /scripts/run.sh +COPY hack/make/.build-empty-images /scripts/build-empty-images.sh COPY integration/testdata /tests/integration/testdata COPY integration/build/testdata /tests/integration/build/testdata diff --git a/hack/make/.ensure-emptyfs b/hack/make/.build-empty-images similarity index 100% rename from hack/make/.ensure-emptyfs rename to hack/make/.build-empty-images diff --git a/hack/make/.integration-daemon-setup b/hack/make/.integration-daemon-setup index c130e23560a77..4bcc816c2c3e8 100644 --- a/hack/make/.integration-daemon-setup +++ b/hack/make/.integration-daemon-setup @@ -3,5 +3,5 @@ set -e source "$MAKEDIR/.detect-daemon-osarch" if [ "$DOCKER_ENGINE_GOOS" != "windows" ]; then - bundle .ensure-emptyfs + bundle .build-empty-images fi diff --git a/hack/test/e2e-run.sh b/hack/test/e2e-run.sh index 57127c0d18080..545504fa0e7c8 100755 --- a/hack/test/e2e-run.sh +++ b/hack/test/e2e-run.sh @@ -81,5 +81,5 @@ set_platform_timeout() { fi } -sh /scripts/ensure-emptyfs.sh +sh /scripts/build-empty-images.sh run_test_integration From 4cc2081119f7cceb6325857ca9020bc7868032d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 22 May 2023 11:48:50 +0200 Subject: [PATCH 037/293] integration: Add TestImageInspectEmptyTagsAndDigests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 6506579e18f7880b2f3db1966fa728f91a102514) Signed-off-by: Paweł Gronowski --- integration/image/inspect_test.go | 41 ++++++++++++++++++++++++++ testutil/environment/clean.go | 3 ++ testutil/environment/protect.go | 1 + testutil/environment/special_images.go | 7 +++++ 4 files changed, 52 insertions(+) create mode 100644 integration/image/inspect_test.go create mode 100644 testutil/environment/special_images.go diff --git a/integration/image/inspect_test.go b/integration/image/inspect_test.go new file mode 100644 index 0000000000000..519e824c47f04 --- /dev/null +++ b/integration/image/inspect_test.go @@ -0,0 +1,41 @@ +package image + +import ( + "context" + "encoding/json" + "testing" + + "github.com/docker/docker/testutil/environment" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" + "gotest.tools/v3/skip" +) + +// Regression test for: https://github.com/moby/moby/issues/45556 +func TestImageInspectEmptyTagsAndDigests(t *testing.T) { + skip.If(t, testEnv.OSType == "windows", "build-empty-images is not called on Windows") + defer setupTest(t)() + + client := testEnv.APIClient() + ctx := context.Background() + + danglingId := environment.DanglingImageIdGraphDriver + if testEnv.UsingSnapshotter() { + danglingId = environment.DanglingImageIdSnapshotter + } + + inspect, raw, err := client.ImageInspectWithRaw(ctx, danglingId) + assert.NilError(t, err) + + // Must be a zero length array, not null. + assert.Check(t, is.Len(inspect.RepoTags, 0)) + assert.Check(t, is.Len(inspect.RepoDigests, 0)) + + var rawJson map[string]interface{} + err = json.Unmarshal(raw, &rawJson) + assert.NilError(t, err) + + // Check if the raw json is also an array, not null. + assert.Check(t, is.Len(rawJson["RepoTags"], 0)) + assert.Check(t, is.Len(rawJson["RepoDigests"], 0)) +} diff --git a/testutil/environment/clean.go b/testutil/environment/clean.go index 74c8339d35a0b..4c780e60cb4c1 100644 --- a/testutil/environment/clean.go +++ b/testutil/environment/clean.go @@ -98,6 +98,9 @@ func deleteAllImages(t testing.TB, apiclient client.ImageAPIClient, protectedIma ctx := context.Background() for _, image := range images { tags := tagsFromImageSummary(image) + if _, ok := protectedImages[image.ID]; ok { + continue + } if len(tags) == 0 { removeImage(ctx, t, apiclient, image.ID) continue diff --git a/testutil/environment/protect.go b/testutil/environment/protect.go index 2a0d5281b6e33..a84dccc9a5b99 100644 --- a/testutil/environment/protect.go +++ b/testutil/environment/protect.go @@ -95,6 +95,7 @@ func ProtectImages(t testing.TB, testEnv *Execution) { images = append(images, frozenImages...) } testEnv.ProtectImage(t, images...) + testEnv.ProtectImage(t, DanglingImageIdGraphDriver, DanglingImageIdSnapshotter) } func getExistingImages(t testing.TB, testEnv *Execution) []string { diff --git a/testutil/environment/special_images.go b/testutil/environment/special_images.go new file mode 100644 index 0000000000000..b486e0498c762 --- /dev/null +++ b/testutil/environment/special_images.go @@ -0,0 +1,7 @@ +package environment + +// Graph driver image store identifies images by the ID of their config. +const DanglingImageIdGraphDriver = "sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43" + +// The containerd image store identifies images by the ID of their manifest/manifest list. +const DanglingImageIdSnapshotter = "sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456" From 329d671aef88f0f5d7a66dd717fbdf353d03c144 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 23 May 2023 12:56:02 +0200 Subject: [PATCH 038/293] Dockerfile: temporarily skip CRIU stage The package repository currently has issues; => ERROR https://download.opensuse.org/repositories/devel:/tools:/criu/Debian_11/Release.key The only test currently using this binary is currently skipped, as the test was broken; https://github.com/moby/moby/blob/6e98a7f2c9184d4e91df8abf06e2245a0cd77c58/integration/container/checkpoint_test.go#L32-L33 So let's disable this stage for the time being. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit d3d2823edfb0247fe3ee320414dc3c836e53a3b9) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2ffcb830d966a..cf73d8525ef25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -447,7 +447,12 @@ COPY --from=tomll /build/ /usr/local/bin/ COPY --from=gowinres /build/ /usr/local/bin/ COPY --from=tini /build/ /usr/local/bin/ COPY --from=registry /build/ /usr/local/bin/ -COPY --from=criu /build/ /usr/local/bin/ + +# Skip the CRIU stage for now, as the opensuse package repository is sometimes +# unstable, and we're currently not using it in CI. +# +# FIXME(thaJeztah): re-enable this stage when https://github.com/moby/moby/issues/38963 is resolved (see https://github.com/moby/moby/pull/38984) +# COPY --from=criu /build/ /usr/local/bin/ COPY --from=gotestsum /build/ /usr/local/bin/ COPY --from=golangci_lint /build/ /usr/local/bin/ COPY --from=shfmt /build/ /usr/local/bin/ From d64bab35eea9f964324e60a11639e6aaff01a3fe Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Wed, 24 May 2023 16:56:17 -0400 Subject: [PATCH 039/293] daemon: lock in snapshotter setting at daemon init Feature flags are one of the configuration items which can be reloaded without restarting the daemon. Whether the daemon uses the containerd snapshotter service or the legacy graph drivers is controlled by a feature flag. However, much of the code which checks the snapshotter feature flag assumes that the flag cannot change at runtime. Make it so that the snapshotter setting can only be changed by restarting the daemon, even if the flag state changes after a live configuration reload. Signed-off-by: Cory Snider (cherry picked from commit 9b9c5242eb6eaeb815c4fcfc1ba7bcb70212a274) Signed-off-by: Sebastiaan van Stijn --- daemon/daemon.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index a2e375a978a8e..9be2f289696af 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -129,6 +129,8 @@ type Daemon struct { // It stores metadata for the content store (used for manifest caching) // This needs to be closed on daemon exit mdDB *bbolt.DB + + usesSnapshotter bool } // ID returns the daemon id @@ -158,16 +160,7 @@ func (daemon *Daemon) Features() *map[string]bool { // UsesSnapshotter returns true if feature flag to use containerd snapshotter is enabled func (daemon *Daemon) UsesSnapshotter() bool { - // TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only. - if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { - return true - } - if daemon.configStore.Features != nil { - if b, ok := daemon.configStore.Features["containerd-snapshotter"]; ok { - return b - } - } - return false + return daemon.usesSnapshotter } // RegistryHosts returns registry configuration in containerd resolvers format @@ -801,6 +794,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S startupDone: make(chan struct{}), } + // TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only. + if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { + d.usesSnapshotter = true + } else { + d.usesSnapshotter = config.Features["containerd-snapshotter"] + } + // Ensure the daemon is properly shutdown if there is a failure during // initialization defer func() { From 04eccf81654771f187cd7fdf34b3b12553e4e028 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 25 May 2023 18:56:52 +0200 Subject: [PATCH 040/293] vendor: github.com/containerd/go-runc v1.1.0 full diff: https://github.com/containerd/go-runc/compare/v1.0.0...v1.1.0 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 3512b0409337b5b8e6623e502985a4362ec424fe) Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 3 +- .../containerd/go-runc/.golangci.yml | 20 ++ .../github.com/containerd/go-runc/.travis.yml | 21 -- .../github.com/containerd/go-runc/README.md | 10 +- .../containerd/go-runc/command_other.go | 2 +- .../github.com/containerd/go-runc/console.go | 9 +- .../github.com/containerd/go-runc/events.go | 17 +- vendor/github.com/containerd/go-runc/io.go | 12 +- .../github.com/containerd/go-runc/io_unix.go | 17 +- .../containerd/go-runc/io_windows.go | 47 +--- .../github.com/containerd/go-runc/monitor.go | 54 ++++- vendor/github.com/containerd/go-runc/runc.go | 217 +++++++++++++----- .../containerd/go-runc/runc_unix.go | 38 --- .../containerd/go-runc/runc_windows.go | 31 --- vendor/github.com/containerd/go-runc/utils.go | 16 +- .../specs-go/features/features.go | 125 ++++++++++ vendor/modules.txt | 5 +- 18 files changed, 414 insertions(+), 232 deletions(-) create mode 100644 vendor/github.com/containerd/go-runc/.golangci.yml delete mode 100644 vendor/github.com/containerd/go-runc/.travis.yml delete mode 100644 vendor/github.com/containerd/go-runc/runc_unix.go delete mode 100644 vendor/github.com/containerd/go-runc/runc_windows.go create mode 100644 vendor/github.com/opencontainers/runtime-spec/specs-go/features/features.go diff --git a/vendor.mod b/vendor.mod index 44d53f8c91cac..1885aeeb7b7ae 100644 --- a/vendor.mod +++ b/vendor.mod @@ -118,7 +118,7 @@ require ( github.com/containerd/cgroups v1.0.4 // indirect github.com/containerd/console v1.0.3 // indirect github.com/containerd/go-cni v1.1.6 // indirect - github.com/containerd/go-runc v1.0.0 // indirect + github.com/containerd/go-runc v1.1.0 // indirect github.com/containerd/nydus-snapshotter v0.3.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.13.0 // indirect github.com/containerd/ttrpc v1.1.1 // indirect diff --git a/vendor.sum b/vendor.sum index 2d78b1393f75b..43c80d2b479ae 100644 --- a/vendor.sum +++ b/vendor.sum @@ -397,8 +397,9 @@ github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.1.0 h1:OX4f+/i2y5sUT7LhmcJH7GYrjjhHa1QI4e8yO0gGleA= +github.com/containerd/go-runc v1.1.0/go.mod h1:xJv2hFF7GvHtTJd9JqTS2UVxMkULUYw4JN5XAUZqH5U= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= diff --git a/vendor/github.com/containerd/go-runc/.golangci.yml b/vendor/github.com/containerd/go-runc/.golangci.yml new file mode 100644 index 0000000000000..240eaed095c59 --- /dev/null +++ b/vendor/github.com/containerd/go-runc/.golangci.yml @@ -0,0 +1,20 @@ +linters: + enable: + - gofmt + - goimports + - ineffassign + - misspell + - revive + - staticcheck + - unconvert + - unused + - vet + disable: + - errcheck + +issues: + include: + - EXC0002 + +run: + timeout: 2m diff --git a/vendor/github.com/containerd/go-runc/.travis.yml b/vendor/github.com/containerd/go-runc/.travis.yml deleted file mode 100644 index 724ee09d24f80..0000000000000 --- a/vendor/github.com/containerd/go-runc/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go -go: - - 1.13.x - - 1.14.x - - 1.15.x - -install: - - go get -t ./... - - go get -u github.com/vbatts/git-validation - - go get -u github.com/kunalkushwaha/ltag - -before_script: - - pushd ..; git clone https://github.com/containerd/project; popd - -script: - - DCO_VERBOSITY=-q ../project/script/validate/dco - - ../project/script/validate/fileheader ../project/ - - go test -v -race -covermode=atomic -coverprofile=coverage.txt ./... - -after_success: - - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/containerd/go-runc/README.md b/vendor/github.com/containerd/go-runc/README.md index c899bdd7ed873..4262c6268a388 100644 --- a/vendor/github.com/containerd/go-runc/README.md +++ b/vendor/github.com/containerd/go-runc/README.md @@ -1,7 +1,7 @@ # go-runc -[![Build Status](https://travis-ci.org/containerd/go-runc.svg?branch=master)](https://travis-ci.org/containerd/go-runc) -[![codecov](https://codecov.io/gh/containerd/go-runc/branch/master/graph/badge.svg)](https://codecov.io/gh/containerd/go-runc) +[![Build Status](https://github.com/containerd/go-runc/workflows/CI/badge.svg)](https://github.com/containerd/go-runc/actions?query=workflow%3ACI) +[![codecov](https://codecov.io/gh/containerd/go-runc/branch/main/graph/badge.svg)](https://codecov.io/gh/containerd/go-runc) This is a package for consuming the [runc](https://github.com/opencontainers/runc) binary in your Go applications. It tries to expose all the settings and features of the runc CLI. If there is something missing then add it, its opensource! @@ -18,8 +18,8 @@ Docs can be found at [godoc.org](https://godoc.org/github.com/containerd/go-runc The go-runc is a containerd sub-project, licensed under the [Apache 2.0 license](./LICENSE). As a containerd sub-project, you will find the: - * [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md), - * [Maintainers](https://github.com/containerd/project/blob/master/MAINTAINERS), - * and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md) + * [Project governance](https://github.com/containerd/project/blob/main/GOVERNANCE.md), + * [Maintainers](https://github.com/containerd/project/blob/main/MAINTAINERS), + * and [Contributing guidelines](https://github.com/containerd/project/blob/main/CONTRIBUTING.md) information in our [`containerd/project`](https://github.com/containerd/project) repository. diff --git a/vendor/github.com/containerd/go-runc/command_other.go b/vendor/github.com/containerd/go-runc/command_other.go index b8fd4b8660a75..a4adbe1e4724b 100644 --- a/vendor/github.com/containerd/go-runc/command_other.go +++ b/vendor/github.com/containerd/go-runc/command_other.go @@ -1,4 +1,4 @@ -// +build !linux +//go:build !linux /* Copyright The containerd Authors. diff --git a/vendor/github.com/containerd/go-runc/console.go b/vendor/github.com/containerd/go-runc/console.go index ff223e4276658..e8dc862a501fc 100644 --- a/vendor/github.com/containerd/go-runc/console.go +++ b/vendor/github.com/containerd/go-runc/console.go @@ -1,4 +1,4 @@ -// +build !windows +//go:build !windows /* Copyright The containerd Authors. @@ -20,7 +20,6 @@ package runc import ( "fmt" - "io/ioutil" "net" "os" "path/filepath" @@ -53,7 +52,7 @@ func NewConsoleSocket(path string) (*Socket, error) { // On Close(), the socket is deleted func NewTempConsoleSocket() (*Socket, error) { runtimeDir := os.Getenv("XDG_RUNTIME_DIR") - dir, err := ioutil.TempDir(runtimeDir, "pty") + dir, err := os.MkdirTemp(runtimeDir, "pty") if err != nil { return nil, err } @@ -70,7 +69,7 @@ func NewTempConsoleSocket() (*Socket, error) { return nil, err } if runtimeDir != "" { - if err := os.Chmod(abs, 0755|os.ModeSticky); err != nil { + if err := os.Chmod(abs, 0o755|os.ModeSticky); err != nil { return nil, err } } @@ -96,7 +95,7 @@ func (c *Socket) Path() string { // locally (it is sent as non-auxiliary data in the same payload). func recvFd(socket *net.UnixConn) (*os.File, error) { const MaxNameLen = 4096 - var oobSpace = unix.CmsgSpace(4) + oobSpace := unix.CmsgSpace(4) name := make([]byte, MaxNameLen) oob := make([]byte, oobSpace) diff --git a/vendor/github.com/containerd/go-runc/events.go b/vendor/github.com/containerd/go-runc/events.go index d610aeb34e849..6584c49761d89 100644 --- a/vendor/github.com/containerd/go-runc/events.go +++ b/vendor/github.com/containerd/go-runc/events.go @@ -16,6 +16,7 @@ package runc +// Event is a struct to pass runc event information type Event struct { // Type are the event type generated by runc // If the type is "error" then check the Err field on the event for @@ -27,20 +28,23 @@ type Event struct { Err error `json:"-"` } +// Stats is statistical information from the runc process type Stats struct { - Cpu Cpu `json:"cpu"` + Cpu Cpu `json:"cpu"` //revive:disable Memory Memory `json:"memory"` Pids Pids `json:"pids"` Blkio Blkio `json:"blkio"` Hugetlb map[string]Hugetlb `json:"hugetlb"` } +// Hugetlb represents the detailed hugetlb component of the statistics data type Hugetlb struct { Usage uint64 `json:"usage,omitempty"` Max uint64 `json:"max,omitempty"` Failcnt uint64 `json:"failcnt"` } +// BlkioEntry represents a block IO entry in the IO stats type BlkioEntry struct { Major uint64 `json:"major,omitempty"` Minor uint64 `json:"minor,omitempty"` @@ -48,6 +52,7 @@ type BlkioEntry struct { Value uint64 `json:"value,omitempty"` } +// Blkio represents the statistical information from block IO devices type Blkio struct { IoServiceBytesRecursive []BlkioEntry `json:"ioServiceBytesRecursive,omitempty"` IoServicedRecursive []BlkioEntry `json:"ioServicedRecursive,omitempty"` @@ -59,17 +64,22 @@ type Blkio struct { SectorsRecursive []BlkioEntry `json:"sectorsRecursive,omitempty"` } +// Pids represents the process ID information type Pids struct { Current uint64 `json:"current,omitempty"` Limit uint64 `json:"limit,omitempty"` } +// Throttling represents the throttling statistics type Throttling struct { Periods uint64 `json:"periods,omitempty"` ThrottledPeriods uint64 `json:"throttledPeriods,omitempty"` ThrottledTime uint64 `json:"throttledTime,omitempty"` } +// CpuUsage represents the CPU usage statistics +// +//revive:disable-next-line type CpuUsage struct { // Units: nanoseconds. Total uint64 `json:"total,omitempty"` @@ -78,11 +88,15 @@ type CpuUsage struct { User uint64 `json:"user"` } +// Cpu represents the CPU usage and throttling statistics +// +//revive:disable-next-line type Cpu struct { Usage CpuUsage `json:"usage,omitempty"` Throttling Throttling `json:"throttling,omitempty"` } +// MemoryEntry represents an item in the memory use/statistics type MemoryEntry struct { Limit uint64 `json:"limit"` Usage uint64 `json:"usage,omitempty"` @@ -90,6 +104,7 @@ type MemoryEntry struct { Failcnt uint64 `json:"failcnt"` } +// Memory represents the collection of memory statistics from the process type Memory struct { Cache uint64 `json:"cache,omitempty"` Usage MemoryEntry `json:"usage,omitempty"` diff --git a/vendor/github.com/containerd/go-runc/io.go b/vendor/github.com/containerd/go-runc/io.go index 6cf0410c9d33e..3560c69bd332f 100644 --- a/vendor/github.com/containerd/go-runc/io.go +++ b/vendor/github.com/containerd/go-runc/io.go @@ -22,6 +22,7 @@ import ( "os/exec" ) +// IO is the terminal IO interface type IO interface { io.Closer Stdin() io.WriteCloser @@ -30,6 +31,7 @@ type IO interface { Set(*exec.Cmd) } +// StartCloser is an interface to handle IO closure after start type StartCloser interface { CloseAfterStart() error } @@ -76,6 +78,12 @@ func (p *pipe) Close() error { return err } +// NewPipeIO creates pipe pairs to be used with runc. It is not implemented +// on Windows. +func NewPipeIO(uid, gid int, opts ...IOOpt) (i IO, err error) { + return newPipeIO(uid, gid, opts...) +} + type pipeIO struct { in *pipe out *pipe @@ -144,12 +152,12 @@ func (i *pipeIO) Set(cmd *exec.Cmd) { } } +// NewSTDIO returns I/O setup for standard OS in/out/err usage func NewSTDIO() (IO, error) { return &stdio{}, nil } -type stdio struct { -} +type stdio struct{} func (s *stdio) Close() error { return nil diff --git a/vendor/github.com/containerd/go-runc/io_unix.go b/vendor/github.com/containerd/go-runc/io_unix.go index ccf1dd490d9ec..83e3667a9ae55 100644 --- a/vendor/github.com/containerd/go-runc/io_unix.go +++ b/vendor/github.com/containerd/go-runc/io_unix.go @@ -1,4 +1,4 @@ -// +build !windows +//go:build !windows /* Copyright The containerd Authors. @@ -19,14 +19,15 @@ package runc import ( - "github.com/pkg/errors" + "fmt" + "runtime" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" - "runtime" ) -// NewPipeIO creates pipe pairs to be used with runc -func NewPipeIO(uid, gid int, opts ...IOOpt) (i IO, err error) { +// newPipeIO creates pipe pairs to be used with runc +func newPipeIO(uid, gid int, opts ...IOOpt) (i IO, err error) { option := defaultIOOption() for _, o := range opts { o(option) @@ -54,7 +55,7 @@ func NewPipeIO(uid, gid int, opts ...IOOpt) (i IO, err error) { if runtime.GOOS == "darwin" { logrus.WithError(err).Debug("failed to chown stdin, ignored") } else { - return nil, errors.Wrap(err, "failed to chown stdin") + return nil, fmt.Errorf("failed to chown stdin: %w", err) } } } @@ -69,7 +70,7 @@ func NewPipeIO(uid, gid int, opts ...IOOpt) (i IO, err error) { if runtime.GOOS == "darwin" { logrus.WithError(err).Debug("failed to chown stdout, ignored") } else { - return nil, errors.Wrap(err, "failed to chown stdout") + return nil, fmt.Errorf("failed to chown stdout: %w", err) } } } @@ -84,7 +85,7 @@ func NewPipeIO(uid, gid int, opts ...IOOpt) (i IO, err error) { if runtime.GOOS == "darwin" { logrus.WithError(err).Debug("failed to chown stderr, ignored") } else { - return nil, errors.Wrap(err, "failed to chown stderr") + return nil, fmt.Errorf("failed to chown stderr: %w", err) } } } diff --git a/vendor/github.com/containerd/go-runc/io_windows.go b/vendor/github.com/containerd/go-runc/io_windows.go index fc56ac4f301a0..a433f40ba7be5 100644 --- a/vendor/github.com/containerd/go-runc/io_windows.go +++ b/vendor/github.com/containerd/go-runc/io_windows.go @@ -1,4 +1,4 @@ -// +build windows +//go:build windows /* Copyright The containerd Authors. @@ -18,45 +18,8 @@ package runc -// NewPipeIO creates pipe pairs to be used with runc -func NewPipeIO(opts ...IOOpt) (i IO, err error) { - option := defaultIOOption() - for _, o := range opts { - o(option) - } - var ( - pipes []*pipe - stdin, stdout, stderr *pipe - ) - // cleanup in case of an error - defer func() { - if err != nil { - for _, p := range pipes { - p.Close() - } - } - }() - if option.OpenStdin { - if stdin, err = newPipe(); err != nil { - return nil, err - } - pipes = append(pipes, stdin) - } - if option.OpenStdout { - if stdout, err = newPipe(); err != nil { - return nil, err - } - pipes = append(pipes, stdout) - } - if option.OpenStderr { - if stderr, err = newPipe(); err != nil { - return nil, err - } - pipes = append(pipes, stderr) - } - return &pipeIO{ - in: stdin, - out: stdout, - err: stderr, - }, nil +import "errors" + +func newPipeIO(uid, gid int, opts ...IOOpt) (i IO, err error) { + return nil, errors.New("not implemented on Windows") } diff --git a/vendor/github.com/containerd/go-runc/monitor.go b/vendor/github.com/containerd/go-runc/monitor.go index ff06a3fca93e9..b9938add61392 100644 --- a/vendor/github.com/containerd/go-runc/monitor.go +++ b/vendor/github.com/containerd/go-runc/monitor.go @@ -18,32 +18,37 @@ package runc import ( "os/exec" + "runtime" "syscall" "time" ) +// Monitor is the default ProcessMonitor for handling runc process exit var Monitor ProcessMonitor = &defaultMonitor{} +// Exit holds the exit information from a process type Exit struct { Timestamp time.Time Pid int Status int } -// ProcessMonitor is an interface for process monitoring +// ProcessMonitor is an interface for process monitoring. // // It allows daemons using go-runc to have a SIGCHLD handler // to handle exits without introducing races between the handler -// and go's exec.Cmd -// These methods should match the methods exposed by exec.Cmd to provide -// a consistent experience for the caller +// and go's exec.Cmd. +// +// ProcessMonitor also provides a StartLocked method which is similar to +// Start, but locks the goroutine used to start the process to an OS thread +// (for example: when Pdeathsig is set). type ProcessMonitor interface { Start(*exec.Cmd) (chan Exit, error) + StartLocked(*exec.Cmd) (chan Exit, error) Wait(*exec.Cmd, chan Exit) (int, error) } -type defaultMonitor struct { -} +type defaultMonitor struct{} func (m *defaultMonitor) Start(c *exec.Cmd) (chan Exit, error) { if err := c.Start(); err != nil { @@ -70,6 +75,43 @@ func (m *defaultMonitor) Start(c *exec.Cmd) (chan Exit, error) { return ec, nil } +// StartLocked is like Start, but locks the goroutine used to start the process to +// the OS thread for use-cases where the parent thread matters to the child process +// (for example: when Pdeathsig is set). +func (m *defaultMonitor) StartLocked(c *exec.Cmd) (chan Exit, error) { + started := make(chan error) + ec := make(chan Exit, 1) + go func() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + if err := c.Start(); err != nil { + started <- err + return + } + close(started) + var status int + if err := c.Wait(); err != nil { + status = 255 + if exitErr, ok := err.(*exec.ExitError); ok { + if ws, ok := exitErr.Sys().(syscall.WaitStatus); ok { + status = ws.ExitStatus() + } + } + } + ec <- Exit{ + Timestamp: time.Now(), + Pid: c.Process.Pid, + Status: status, + } + close(ec) + }() + if err := <-started; err != nil { + return nil, err + } + return ec, nil +} + func (m *defaultMonitor) Wait(c *exec.Cmd, ec chan Exit) (int, error) { e := <-ec return e.Status, nil diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go index f5f03ae95eb4b..61646df6a126d 100644 --- a/vendor/github.com/containerd/go-runc/runc.go +++ b/vendor/github.com/containerd/go-runc/runc.go @@ -23,21 +23,22 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "os/exec" "path/filepath" "strconv" "strings" + "syscall" "time" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-spec/specs-go/features" ) -// Format is the type of log formatting options avaliable +// Format is the type of log formatting options available type Format string -// TopBody represents the structured data of the full ps output +// TopResults represents the structured data of the full ps output type TopResults struct { // Processes running in the container, where each is process is an array of values corresponding to the headers Processes [][]string `json:"Processes"` @@ -48,15 +49,53 @@ type TopResults struct { const ( none Format = "" + // JSON represents the JSON format JSON Format = "json" + // Text represents plain text format Text Format = "text" - // DefaultCommand is the default command for Runc - DefaultCommand = "runc" ) +// DefaultCommand is the default command for Runc +var DefaultCommand = "runc" + +// Runc is the client to the runc cli +type Runc struct { + // Command overrides the name of the runc binary. If empty, DefaultCommand + // is used. + Command string + Root string + Debug bool + Log string + LogFormat Format + // PdeathSignal sets a signal the child process will receive when the + // parent dies. + // + // When Pdeathsig is set, command invocations will call runtime.LockOSThread + // to prevent OS thread termination from spuriously triggering the + // signal. See https://github.com/golang/go/issues/27505 and + // https://github.com/golang/go/blob/126c22a09824a7b52c019ed9a1d198b4e7781676/src/syscall/exec_linux.go#L48-L51 + // + // A program with GOMAXPROCS=1 might hang because of the use of + // runtime.LockOSThread. Callers should ensure they retain at least one + // unlocked thread. + PdeathSignal syscall.Signal // using syscall.Signal to allow compilation on non-unix (unix.Syscall is an alias for syscall.Signal) + Setpgid bool + + // Criu sets the path to the criu binary used for checkpoint and restore. + // + // Deprecated: runc option --criu is now ignored (with a warning), and the + // option will be removed entirely in a future release. Users who need a non- + // standard criu binary should rely on the standard way of looking up binaries + // in $PATH. + Criu string + SystemdCgroup bool + Rootless *bool // nil stands for "auto" + ExtraArgs []string +} + // List returns all containers created inside the provided runc root directory func (r *Runc) List(context context.Context) ([]*Container, error) { - data, err := cmdOutput(r.command(context, "list", "--format=json"), false, nil) + data, err := r.cmdOutput(r.command(context, "list", "--format=json"), false, nil) defer putBuf(data) if err != nil { return nil, err @@ -70,7 +109,7 @@ func (r *Runc) List(context context.Context) ([]*Container, error) { // State returns the state for the container provided by id func (r *Runc) State(context context.Context, id string) (*Container, error) { - data, err := cmdOutput(r.command(context, "state", id), true, nil) + data, err := r.cmdOutput(r.command(context, "state", id), true, nil) defer putBuf(data) if err != nil { return nil, fmt.Errorf("%s: %s", err, data.String()) @@ -82,10 +121,12 @@ func (r *Runc) State(context context.Context, id string) (*Container, error) { return &c, nil } +// ConsoleSocket handles the path of the socket for console access type ConsoleSocket interface { Path() string } +// CreateOpts holds all the options information for calling runc with supported options type CreateOpts struct { IO // PidFile is a path to where a pid file should be created @@ -96,6 +137,7 @@ type CreateOpts struct { NoNewKeyring bool ExtraFiles []*os.File Started chan<- int + ExtraArgs []string } func (o *CreateOpts) args() (out []string, err error) { @@ -121,38 +163,50 @@ func (o *CreateOpts) args() (out []string, err error) { if o.ExtraFiles != nil { out = append(out, "--preserve-fds", strconv.Itoa(len(o.ExtraFiles))) } + if len(o.ExtraArgs) > 0 { + out = append(out, o.ExtraArgs...) + } return out, nil } +func (r *Runc) startCommand(cmd *exec.Cmd) (chan Exit, error) { + if r.PdeathSignal != 0 { + return Monitor.StartLocked(cmd) + } + return Monitor.Start(cmd) +} + // Create creates a new container and returns its pid if it was created successfully func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error { args := []string{"create", "--bundle", bundle} - if opts != nil { - oargs, err := opts.args() - if err != nil { - return err - } - args = append(args, oargs...) + if opts == nil { + opts = &CreateOpts{} + } + + oargs, err := opts.args() + if err != nil { + return err } + args = append(args, oargs...) cmd := r.command(context, append(args, id)...) - if opts != nil && opts.IO != nil { + if opts.IO != nil { opts.Set(cmd) } cmd.ExtraFiles = opts.ExtraFiles if cmd.Stdout == nil && cmd.Stderr == nil { - data, err := cmdOutput(cmd, true, nil) + data, err := r.cmdOutput(cmd, true, nil) defer putBuf(data) if err != nil { return fmt.Errorf("%s: %s", err, data.String()) } return nil } - ec, err := Monitor.Start(cmd) + ec, err := r.startCommand(cmd) if err != nil { return err } - if opts != nil && opts.IO != nil { + if opts.IO != nil { if c, ok := opts.IO.(StartCloser); ok { if err := c.CloseAfterStart(); err != nil { return err @@ -171,12 +225,14 @@ func (r *Runc) Start(context context.Context, id string) error { return r.runOrError(r.command(context, "start", id)) } +// ExecOpts holds optional settings when starting an exec process with runc type ExecOpts struct { IO PidFile string ConsoleSocket ConsoleSocket Detach bool Started chan<- int + ExtraArgs []string } func (o *ExecOpts) args() (out []string, err error) { @@ -193,16 +249,22 @@ func (o *ExecOpts) args() (out []string, err error) { } out = append(out, "--pid-file", abs) } + if len(o.ExtraArgs) > 0 { + out = append(out, o.ExtraArgs...) + } return out, nil } // Exec executes an additional process inside the container based on a full // OCI Process specification func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error { + if opts == nil { + opts = &ExecOpts{} + } if opts.Started != nil { defer close(opts.Started) } - f, err := ioutil.TempFile(os.Getenv("XDG_RUNTIME_DIR"), "runc-process") + f, err := os.CreateTemp(os.Getenv("XDG_RUNTIME_DIR"), "runc-process") if err != nil { return err } @@ -213,33 +275,31 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts return err } args := []string{"exec", "--process", f.Name()} - if opts != nil { - oargs, err := opts.args() - if err != nil { - return err - } - args = append(args, oargs...) + oargs, err := opts.args() + if err != nil { + return err } + args = append(args, oargs...) cmd := r.command(context, append(args, id)...) - if opts != nil && opts.IO != nil { + if opts.IO != nil { opts.Set(cmd) } if cmd.Stdout == nil && cmd.Stderr == nil { - data, err := cmdOutput(cmd, true, opts.Started) + data, err := r.cmdOutput(cmd, true, opts.Started) defer putBuf(data) if err != nil { return fmt.Errorf("%w: %s", err, data.String()) } return nil } - ec, err := Monitor.Start(cmd) + ec, err := r.startCommand(cmd) if err != nil { return err } if opts.Started != nil { opts.Started <- cmd.Process.Pid } - if opts != nil && opts.IO != nil { + if opts.IO != nil { if c, ok := opts.IO.(StartCloser); ok { if err := c.CloseAfterStart(); err != nil { return err @@ -256,22 +316,24 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts // Run runs the create, start, delete lifecycle of the container // and returns its exit status after it has exited func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) (int, error) { + if opts == nil { + opts = &CreateOpts{} + } if opts.Started != nil { defer close(opts.Started) } args := []string{"run", "--bundle", bundle} - if opts != nil { - oargs, err := opts.args() - if err != nil { - return -1, err - } - args = append(args, oargs...) + oargs, err := opts.args() + if err != nil { + return -1, err } + args = append(args, oargs...) cmd := r.command(context, append(args, id)...) - if opts != nil && opts.IO != nil { + if opts.IO != nil { opts.Set(cmd) } - ec, err := Monitor.Start(cmd) + cmd.ExtraFiles = opts.ExtraFiles + ec, err := r.startCommand(cmd) if err != nil { return -1, err } @@ -285,14 +347,19 @@ func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) return status, err } +// DeleteOpts holds the deletion options for calling `runc delete` type DeleteOpts struct { - Force bool + Force bool + ExtraArgs []string } func (o *DeleteOpts) args() (out []string) { if o.Force { out = append(out, "--force") } + if len(o.ExtraArgs) > 0 { + out = append(out, o.ExtraArgs...) + } return out } @@ -307,13 +374,17 @@ func (r *Runc) Delete(context context.Context, id string, opts *DeleteOpts) erro // KillOpts specifies options for killing a container and its processes type KillOpts struct { - All bool + All bool + ExtraArgs []string } func (o *KillOpts) args() (out []string) { if o.All { out = append(out, "--all") } + if len(o.ExtraArgs) > 0 { + out = append(out, o.ExtraArgs...) + } return out } @@ -335,7 +406,7 @@ func (r *Runc) Stats(context context.Context, id string) (*Stats, error) { if err != nil { return nil, err } - ec, err := Monitor.Start(cmd) + ec, err := r.startCommand(cmd) if err != nil { return nil, err } @@ -357,7 +428,7 @@ func (r *Runc) Events(context context.Context, id string, interval time.Duration if err != nil { return nil, err } - ec, err := Monitor.Start(cmd) + ec, err := r.startCommand(cmd) if err != nil { rd.Close() return nil, err @@ -401,7 +472,7 @@ func (r *Runc) Resume(context context.Context, id string) error { // Ps lists all the processes inside the container returning their pids func (r *Runc) Ps(context context.Context, id string) ([]int, error) { - data, err := cmdOutput(r.command(context, "ps", "--format", "json", id), true, nil) + data, err := r.cmdOutput(r.command(context, "ps", "--format", "json", id), true, nil) defer putBuf(data) if err != nil { return nil, fmt.Errorf("%s: %s", err, data.String()) @@ -415,7 +486,7 @@ func (r *Runc) Ps(context context.Context, id string) ([]int, error) { // Top lists all the processes inside the container returning the full ps data func (r *Runc) Top(context context.Context, id string, psOptions string) (*TopResults, error) { - data, err := cmdOutput(r.command(context, "ps", "--format", "table", id, psOptions), true, nil) + data, err := r.cmdOutput(r.command(context, "ps", "--format", "table", id, psOptions), true, nil) defer putBuf(data) if err != nil { return nil, fmt.Errorf("%s: %s", err, data.String()) @@ -428,6 +499,7 @@ func (r *Runc) Top(context context.Context, id string, psOptions string) (*TopRe return topResults, nil } +// CheckpointOpts holds the options for performing a criu checkpoint using runc type CheckpointOpts struct { // ImagePath is the path for saving the criu image file ImagePath string @@ -454,13 +526,18 @@ type CheckpointOpts struct { LazyPages bool // StatusFile is the file criu writes \0 to once lazy-pages is ready StatusFile *os.File + ExtraArgs []string } +// CgroupMode defines the cgroup mode used for checkpointing type CgroupMode string const ( - Soft CgroupMode = "soft" - Full CgroupMode = "full" + // Soft is the "soft" cgroup mode + Soft CgroupMode = "soft" + // Full is the "full" cgroup mode + Full CgroupMode = "full" + // Strict is the "strict" cgroup mode Strict CgroupMode = "strict" ) @@ -498,9 +575,13 @@ func (o *CheckpointOpts) args() (out []string) { if o.LazyPages { out = append(out, "--lazy-pages") } + if len(o.ExtraArgs) > 0 { + out = append(out, o.ExtraArgs...) + } return out } +// CheckpointAction represents specific actions executed during checkpoint/restore type CheckpointAction func([]string) []string // LeaveRunning keeps the container running after the checkpoint has been completed @@ -535,6 +616,7 @@ func (r *Runc) Checkpoint(context context.Context, id string, opts *CheckpointOp return r.runOrError(cmd) } +// RestoreOpts holds the options for performing a criu restore using runc type RestoreOpts struct { CheckpointOpts IO @@ -544,6 +626,7 @@ type RestoreOpts struct { NoSubreaper bool NoPivot bool ConsoleSocket ConsoleSocket + ExtraArgs []string } func (o *RestoreOpts) args() ([]string, error) { @@ -567,6 +650,9 @@ func (o *RestoreOpts) args() ([]string, error) { if o.NoSubreaper { out = append(out, "-no-subreaper") } + if len(o.ExtraArgs) > 0 { + out = append(out, o.ExtraArgs...) + } return out, nil } @@ -585,7 +671,7 @@ func (r *Runc) Restore(context context.Context, id, bundle string, opts *Restore if opts != nil && opts.IO != nil { opts.Set(cmd) } - ec, err := Monitor.Start(cmd) + ec, err := r.startCommand(cmd) if err != nil { return -1, err } @@ -611,14 +697,16 @@ func (r *Runc) Update(context context.Context, id string, resources *specs.Linux if err := json.NewEncoder(buf).Encode(resources); err != nil { return err } - args := []string{"update", "--resources", "-", id} + args := []string{"update", "--resources=-", id} cmd := r.command(context, args...) cmd.Stdin = buf return r.runOrError(cmd) } +// ErrParseRuncVersion is used when the runc version can't be parsed var ErrParseRuncVersion = errors.New("unable to parse runc version") +// Version represents the runc version information type Version struct { Runc string Commit string @@ -627,7 +715,7 @@ type Version struct { // Version returns the runc and runtime-spec versions func (r *Runc) Version(context context.Context) (Version, error) { - data, err := cmdOutput(r.command(context, "--version"), false, nil) + data, err := r.cmdOutput(r.command(context, "--version"), false, nil) defer putBuf(data) if err != nil { return Version{}, err @@ -657,6 +745,26 @@ func parseVersion(data []byte) (Version, error) { return v, nil } +// Features shows the features implemented by the runtime. +// +// Availability: +// +// - runc: supported since runc v1.1.0 +// - crun: https://github.com/containers/crun/issues/1177 +// - youki: https://github.com/containers/youki/issues/815 +func (r *Runc) Features(context context.Context) (*features.Features, error) { + data, err := r.cmdOutput(r.command(context, "features"), false, nil) + defer putBuf(data) + if err != nil { + return nil, err + } + var feat features.Features + if err := json.Unmarshal(data.Bytes(), &feat); err != nil { + return nil, err + } + return &feat, nil +} + func (r *Runc) args() (out []string) { if r.Root != "" { out = append(out, "--root", r.Root) @@ -670,9 +778,6 @@ func (r *Runc) args() (out []string) { if r.LogFormat != none { out = append(out, "--log-format", string(r.LogFormat)) } - if r.Criu != "" { - out = append(out, "--criu", r.Criu) - } if r.SystemdCgroup { out = append(out, "--systemd-cgroup") } @@ -680,6 +785,9 @@ func (r *Runc) args() (out []string) { // nil stands for "auto" (differs from explicit "false") out = append(out, "--rootless="+strconv.FormatBool(*r.Rootless)) } + if len(r.ExtraArgs) > 0 { + out = append(out, r.ExtraArgs...) + } return out } @@ -689,7 +797,7 @@ func (r *Runc) args() (out []string) { // func (r *Runc) runOrError(cmd *exec.Cmd) error { if cmd.Stdout != nil || cmd.Stderr != nil { - ec, err := Monitor.Start(cmd) + ec, err := r.startCommand(cmd) if err != nil { return err } @@ -699,7 +807,7 @@ func (r *Runc) runOrError(cmd *exec.Cmd) error { } return err } - data, err := cmdOutput(cmd, true, nil) + data, err := r.cmdOutput(cmd, true, nil) defer putBuf(data) if err != nil { return fmt.Errorf("%s: %s", err, data.String()) @@ -709,14 +817,14 @@ func (r *Runc) runOrError(cmd *exec.Cmd) error { // callers of cmdOutput are expected to call putBuf on the returned Buffer // to ensure it is released back to the shared pool after use. -func cmdOutput(cmd *exec.Cmd, combined bool, started chan<- int) (*bytes.Buffer, error) { +func (r *Runc) cmdOutput(cmd *exec.Cmd, combined bool, started chan<- int) (*bytes.Buffer, error) { b := getBuf() cmd.Stdout = b if combined { cmd.Stderr = b } - ec, err := Monitor.Start(cmd) + ec, err := r.startCommand(cmd) if err != nil { return nil, err } @@ -732,6 +840,7 @@ func cmdOutput(cmd *exec.Cmd, combined bool, started chan<- int) (*bytes.Buffer, return b, err } +// ExitError holds the status return code when a process exits with an error code type ExitError struct { Status int } diff --git a/vendor/github.com/containerd/go-runc/runc_unix.go b/vendor/github.com/containerd/go-runc/runc_unix.go deleted file mode 100644 index 548ffd6b90c64..0000000000000 --- a/vendor/github.com/containerd/go-runc/runc_unix.go +++ /dev/null @@ -1,38 +0,0 @@ -//+build !windows - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package runc - -import ( - "golang.org/x/sys/unix" -) - -// Runc is the client to the runc cli -type Runc struct { - //If command is empty, DefaultCommand is used - Command string - Root string - Debug bool - Log string - LogFormat Format - PdeathSignal unix.Signal - Setpgid bool - Criu string - SystemdCgroup bool - Rootless *bool // nil stands for "auto" -} diff --git a/vendor/github.com/containerd/go-runc/runc_windows.go b/vendor/github.com/containerd/go-runc/runc_windows.go deleted file mode 100644 index c5873de8b6f7f..0000000000000 --- a/vendor/github.com/containerd/go-runc/runc_windows.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package runc - -// Runc is the client to the runc cli -type Runc struct { - //If command is empty, DefaultCommand is used - Command string - Root string - Debug bool - Log string - LogFormat Format - Setpgid bool - Criu string - SystemdCgroup bool - Rootless *bool // nil stands for "auto" -} diff --git a/vendor/github.com/containerd/go-runc/utils.go b/vendor/github.com/containerd/go-runc/utils.go index 948b6336a7f02..4f39f6ec1aba2 100644 --- a/vendor/github.com/containerd/go-runc/utils.go +++ b/vendor/github.com/containerd/go-runc/utils.go @@ -18,34 +18,22 @@ package runc import ( "bytes" - "io/ioutil" + "os" "strconv" "strings" "sync" - "syscall" ) // ReadPidFile reads the pid file at the provided path and returns // the pid or an error if the read and conversion is unsuccessful func ReadPidFile(path string) (int, error) { - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return -1, err } return strconv.Atoi(string(data)) } -const exitSignalOffset = 128 - -// exitStatus returns the correct exit status for a process based on if it -// was signaled or exited cleanly -func exitStatus(status syscall.WaitStatus) int { - if status.Signaled() { - return exitSignalOffset + int(status.Signal()) - } - return status.ExitStatus() -} - var bytesBufferPool = sync.Pool{ New: func() interface{} { return bytes.NewBuffer(nil) diff --git a/vendor/github.com/opencontainers/runtime-spec/specs-go/features/features.go b/vendor/github.com/opencontainers/runtime-spec/specs-go/features/features.go new file mode 100644 index 0000000000000..230e88f568e96 --- /dev/null +++ b/vendor/github.com/opencontainers/runtime-spec/specs-go/features/features.go @@ -0,0 +1,125 @@ +// Package features provides the Features struct. +package features + +// Features represents the supported features of the runtime. +type Features struct { + // OCIVersionMin is the minimum OCI Runtime Spec version recognized by the runtime, e.g., "1.0.0". + OCIVersionMin string `json:"ociVersionMin,omitempty"` + + // OCIVersionMax is the maximum OCI Runtime Spec version recognized by the runtime, e.g., "1.0.2-dev". + OCIVersionMax string `json:"ociVersionMax,omitempty"` + + // Hooks is the list of the recognized hook names, e.g., "createRuntime". + // Nil value means "unknown", not "no support for any hook". + Hooks []string `json:"hooks,omitempty"` + + // MountOptions is the list of the recognized mount options, e.g., "ro". + // Nil value means "unknown", not "no support for any mount option". + // This list does not contain filesystem-specific options passed to mount(2) syscall as (const void *). + MountOptions []string `json:"mountOptions,omitempty"` + + // Linux is specific to Linux. + Linux *Linux `json:"linux,omitempty"` + + // Annotations contains implementation-specific annotation strings, + // such as the implementation version, and third-party extensions. + Annotations map[string]string `json:"annotations,omitempty"` +} + +// Linux is specific to Linux. +type Linux struct { + // Namespaces is the list of the recognized namespaces, e.g., "mount". + // Nil value means "unknown", not "no support for any namespace". + Namespaces []string `json:"namespaces,omitempty"` + + // Capabilities is the list of the recognized capabilities , e.g., "CAP_SYS_ADMIN". + // Nil value means "unknown", not "no support for any capability". + Capabilities []string `json:"capabilities,omitempty"` + + Cgroup *Cgroup `json:"cgroup,omitempty"` + Seccomp *Seccomp `json:"seccomp,omitempty"` + Apparmor *Apparmor `json:"apparmor,omitempty"` + Selinux *Selinux `json:"selinux,omitempty"` + IntelRdt *IntelRdt `json:"intelRdt,omitempty"` +} + +// Cgroup represents the "cgroup" field. +type Cgroup struct { + // V1 represents whether Cgroup v1 support is compiled in. + // Unrelated to whether the host uses cgroup v1 or not. + // Nil value means "unknown", not "false". + V1 *bool `json:"v1,omitempty"` + + // V2 represents whether Cgroup v2 support is compiled in. + // Unrelated to whether the host uses cgroup v2 or not. + // Nil value means "unknown", not "false". + V2 *bool `json:"v2,omitempty"` + + // Systemd represents whether systemd-cgroup support is compiled in. + // Unrelated to whether the host uses systemd or not. + // Nil value means "unknown", not "false". + Systemd *bool `json:"systemd,omitempty"` + + // SystemdUser represents whether user-scoped systemd-cgroup support is compiled in. + // Unrelated to whether the host uses systemd or not. + // Nil value means "unknown", not "false". + SystemdUser *bool `json:"systemdUser,omitempty"` + + // Rdma represents whether RDMA cgroup support is compiled in. + // Unrelated to whether the host supports RDMA or not. + // Nil value means "unknown", not "false". + Rdma *bool `json:"rdma,omitempty"` +} + +// Seccomp represents the "seccomp" field. +type Seccomp struct { + // Enabled is true if seccomp support is compiled in. + // Nil value means "unknown", not "false". + Enabled *bool `json:"enabled,omitempty"` + + // Actions is the list of the recognized actions, e.g., "SCMP_ACT_NOTIFY". + // Nil value means "unknown", not "no support for any action". + Actions []string `json:"actions,omitempty"` + + // Operators is the list of the recognized operators, e.g., "SCMP_CMP_NE". + // Nil value means "unknown", not "no support for any operator". + Operators []string `json:"operators,omitempty"` + + // Archs is the list of the recognized archs, e.g., "SCMP_ARCH_X86_64". + // Nil value means "unknown", not "no support for any arch". + Archs []string `json:"archs,omitempty"` + + // KnownFlags is the list of the recognized filter flags, e.g., "SECCOMP_FILTER_FLAG_LOG". + // Nil value means "unknown", not "no flags are recognized". + KnownFlags []string `json:"knownFlags,omitempty"` + + // SupportedFlags is the list of the supported filter flags, e.g., "SECCOMP_FILTER_FLAG_LOG". + // This list may be a subset of KnownFlags due to some flags + // not supported by the current kernel and/or libseccomp. + // Nil value means "unknown", not "no flags are supported". + SupportedFlags []string `json:"supportedFlags,omitempty"` +} + +// Apparmor represents the "apparmor" field. +type Apparmor struct { + // Enabled is true if AppArmor support is compiled in. + // Unrelated to whether the host supports AppArmor or not. + // Nil value means "unknown", not "false". + Enabled *bool `json:"enabled,omitempty"` +} + +// Selinux represents the "selinux" field. +type Selinux struct { + // Enabled is true if SELinux support is compiled in. + // Unrelated to whether the host supports SELinux or not. + // Nil value means "unknown", not "false". + Enabled *bool `json:"enabled,omitempty"` +} + +// IntelRdt represents the "intelRdt" field. +type IntelRdt struct { + // Enabled is true if Intel RDT support is compiled in. + // Unrelated to whether the host supports Intel RDT or not. + // Nil value means "unknown", not "false". + Enabled *bool `json:"enabled,omitempty"` +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d8e17fb540ab8..e5e14460c3382 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -311,8 +311,8 @@ github.com/containerd/fifo # github.com/containerd/go-cni v1.1.6 ## explicit; go 1.17 github.com/containerd/go-cni -# github.com/containerd/go-runc v1.0.0 -## explicit; go 1.13 +# github.com/containerd/go-runc v1.1.0 +## explicit; go 1.18 github.com/containerd/go-runc # github.com/containerd/nydus-snapshotter v0.3.1 ## explicit; go 1.17 @@ -849,6 +849,7 @@ github.com/opencontainers/runc/libcontainer/userns # github.com/opencontainers/runtime-spec v1.1.0-rc.2 ## explicit github.com/opencontainers/runtime-spec/specs-go +github.com/opencontainers/runtime-spec/specs-go/features # github.com/opencontainers/selinux v1.11.0 ## explicit; go 1.19 github.com/opencontainers/selinux/go-selinux From 0d9acd24fe3a4d45b602f896d091a3855057d31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 25 May 2023 14:59:37 +0200 Subject: [PATCH 041/293] c8d/inspect: Fill `Created` time if available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit b9b8b6597a3c0fbe27c85e4090ed50f2d9eef6f3) --- daemon/containerd/image.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index 8ced6436f5be3..ee242132e1721 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -68,14 +68,17 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima exposedPorts[nat.Port(k)] = v } + derefTimeSafely := func(t *time.Time) time.Time { + if t != nil { + return *t + } + return time.Time{} + } + var imgHistory []image.History for _, h := range ociimage.History { - var created time.Time - if h.Created != nil { - created = *h.Created - } imgHistory = append(imgHistory, image.History{ - Created: created, + Created: derefTimeSafely(h.Created), Author: h.Author, CreatedBy: h.CreatedBy, Comment: h.Comment, @@ -88,6 +91,7 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima ID: string(desc.Digest), OS: ociimage.OS, Architecture: ociimage.Architecture, + Created: derefTimeSafely(ociimage.Created), Config: &containertypes.Config{ Entrypoint: ociimage.Config.Entrypoint, Env: ociimage.Config.Env, From ae6e9333c00dd6bfa674fde77399650841803821 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 30 Apr 2023 11:49:22 +0200 Subject: [PATCH 042/293] vendor: github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f full diff: https://github.com/moby/buildkit/compare/v0.11.6...798ad6b0ce9f2fe86dfb2b0277e6770d0b545871 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 79ca6630d471b556227fdef1e360fb5fa278eac4) Signed-off-by: Sebastiaan van Stijn --- builder/builder-next/worker/worker.go | 2 +- vendor.mod | 4 +-- vendor.sum | 10 +++---- .../exporter/containerimage/writer.go | 22 +++++++------- .../dockerfile/dockerfile2llb/image.go | 11 +++---- .../github.com/moby/buildkit/session/grpc.go | 5 ++++ .../moby/buildkit/session/session.go | 29 ++++++++++++++----- .../solver/llbsolver/provenance/predicate.go | 23 ++++++++++----- vendor/modules.txt | 6 +--- 9 files changed, 62 insertions(+), 50 deletions(-) diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index f38304d884bbd..ef5cc1716b685 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -50,7 +50,7 @@ import ( ) func init() { - version.Version = "v0.11.6" + version.Version = "v0.11.7-0.20230525183624-798ad6b0ce9f" } const labelCreatedAt = "buildkit/createdat" diff --git a/vendor.mod b/vendor.mod index 1885aeeb7b7ae..12568d6b68e21 100644 --- a/vendor.mod +++ b/vendor.mod @@ -56,7 +56,7 @@ require ( github.com/klauspost/compress v1.16.3 github.com/miekg/dns v1.1.43 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible - github.com/moby/buildkit v0.11.6 // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go + github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go github.com/moby/ipvs v1.1.0 github.com/moby/locker v1.0.1 github.com/moby/patternmatcher v0.5.0 @@ -152,8 +152,6 @@ require ( github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect - github.com/onsi/ginkgo/v2 v2.1.4 // indirect - github.com/onsi/gomega v1.20.1 // indirect github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/prometheus/client_model v0.3.0 // indirect diff --git a/vendor.sum b/vendor.sum index 43c80d2b479ae..985ad6d0f59a4 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1042,8 +1042,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ= -github.com/moby/buildkit v0.11.6 h1:VYNdoKk5TVxN7k4RvZgdeM4GOyRvIi4Z8MXOY7xvyUs= -github.com/moby/buildkit v0.11.6/go.mod h1:GCqKfHhz+pddzfgaR7WmHVEE3nKKZMMDPpK8mh3ZLv4= +github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f h1:9wobL03Y6U8azuDLUqYblbUdVU9jpjqecDdW7w4wZtI= +github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f/go.mod h1:GCqKfHhz+pddzfgaR7WmHVEE3nKKZMMDPpK8mh3ZLv4= github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= @@ -1113,9 +1113,8 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1126,9 +1125,8 @@ github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= diff --git a/vendor/github.com/moby/buildkit/exporter/containerimage/writer.go b/vendor/github.com/moby/buildkit/exporter/containerimage/writer.go index 068d86958f8f7..4cccd9db51282 100644 --- a/vendor/github.com/moby/buildkit/exporter/containerimage/writer.go +++ b/vendor/github.com/moby/buildkit/exporter/containerimage/writer.go @@ -574,11 +574,10 @@ func (ic *ImageWriter) Applier() diff.Applier { func defaultImageConfig() ([]byte, error) { pl := platforms.Normalize(platforms.DefaultSpec()) - img := ocispecs.Image{ - Architecture: pl.Architecture, - OS: pl.OS, - Variant: pl.Variant, - } + img := ocispecs.Image{} + img.Architecture = pl.Architecture + img.OS = pl.OS + img.Variant = pl.Variant img.RootFS.Type = "layers" img.Config.WorkingDir = "/" img.Config.Env = []string{"PATH=" + system.DefaultPathEnv(pl.OS)} @@ -587,13 +586,12 @@ func defaultImageConfig() ([]byte, error) { } func attestationsConfig(layers []ocispecs.Descriptor) ([]byte, error) { - img := ocispecs.Image{ - Architecture: intotoPlatform.Architecture, - OS: intotoPlatform.OS, - OSVersion: intotoPlatform.OSVersion, - OSFeatures: intotoPlatform.OSFeatures, - Variant: intotoPlatform.Variant, - } + img := ocispecs.Image{} + img.Architecture = intotoPlatform.Architecture + img.OS = intotoPlatform.OS + img.OSVersion = intotoPlatform.OSVersion + img.OSFeatures = intotoPlatform.OSFeatures + img.Variant = intotoPlatform.Variant img.RootFS.Type = "layers" for _, layer := range layers { img.RootFS.DiffIDs = append(img.RootFS.DiffIDs, digest.Digest(layer.Annotations["containerd.io/uncompressed"])) diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/image.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/image.go index 36b27aa28aba1..5c3bdeec3294f 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/image.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/image.go @@ -20,13 +20,10 @@ func clone(src Image) Image { } func emptyImage(platform ocispecs.Platform) Image { - img := Image{ - Image: ocispecs.Image{ - Architecture: platform.Architecture, - OS: platform.OS, - Variant: platform.Variant, - }, - } + img := Image{} + img.Architecture = platform.Architecture + img.OS = platform.OS + img.Variant = platform.Variant img.RootFS.Type = "layers" img.Config.WorkingDir = "/" img.Config.Env = []string{"PATH=" + system.DefaultPathEnv(platform.OS)} diff --git a/vendor/github.com/moby/buildkit/session/grpc.go b/vendor/github.com/moby/buildkit/session/grpc.go index dd67c69b64664..6fac82e0b08db 100644 --- a/vendor/github.com/moby/buildkit/session/grpc.go +++ b/vendor/github.com/moby/buildkit/session/grpc.go @@ -112,6 +112,11 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func()) } if err != nil { + select { + case <-ctx.Done(): + return + default: + } if failedBefore { bklog.G(ctx).Error("healthcheck failed fatally") return diff --git a/vendor/github.com/moby/buildkit/session/session.go b/vendor/github.com/moby/buildkit/session/session.go index 50cb3b4486192..f56a18730d22e 100644 --- a/vendor/github.com/moby/buildkit/session/session.go +++ b/vendor/github.com/moby/buildkit/session/session.go @@ -4,6 +4,7 @@ import ( "context" "net" "strings" + "sync" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/moby/buildkit/identity" @@ -36,14 +37,16 @@ type Attachable interface { // Session is a long running connection between client and a daemon type Session struct { - id string - name string - sharedKey string - ctx context.Context - cancelCtx func() - done chan struct{} - grpcServer *grpc.Server - conn net.Conn + mu sync.Mutex // synchronizes conn run and close + id string + name string + sharedKey string + ctx context.Context + cancelCtx func() + done chan struct{} + grpcServer *grpc.Server + conn net.Conn + closeCalled bool } // NewSession returns a new long running session @@ -99,6 +102,11 @@ func (s *Session) ID() string { // Run activates the session func (s *Session) Run(ctx context.Context, dialer Dialer) error { + s.mu.Lock() + if s.closeCalled { + s.mu.Unlock() + return nil + } ctx, cancel := context.WithCancel(ctx) s.cancelCtx = cancel s.done = make(chan struct{}) @@ -118,15 +126,18 @@ func (s *Session) Run(ctx context.Context, dialer Dialer) error { } conn, err := dialer(ctx, "h2c", meta) if err != nil { + s.mu.Unlock() return errors.Wrap(err, "failed to dial gRPC") } s.conn = conn + s.mu.Unlock() serve(ctx, s.grpcServer, conn) return nil } // Close closes the session func (s *Session) Close() error { + s.mu.Lock() if s.cancelCtx != nil && s.done != nil { if s.conn != nil { s.conn.Close() @@ -134,6 +145,8 @@ func (s *Session) Close() error { s.grpcServer.Stop() <-s.done } + s.closeCalled = true + s.mu.Unlock() return nil } diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/provenance/predicate.go b/vendor/github.com/moby/buildkit/solver/llbsolver/provenance/predicate.go index a7b5a78cca51a..f2f7c4e2ad4e2 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/provenance/predicate.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/provenance/predicate.go @@ -64,12 +64,15 @@ func slsaMaterials(srcs Sources) ([]slsa.ProvenanceMaterial, error) { if err != nil { return nil, err } - out = append(out, slsa.ProvenanceMaterial{ + material := slsa.ProvenanceMaterial{ URI: uri, - Digest: slsa.DigestSet{ + } + if s.Digest != "" { + material.Digest = slsa.DigestSet{ s.Digest.Algorithm().String(): s.Digest.Hex(), - }, - }) + } + } + out = append(out, material) } for _, s := range srcs.Git { @@ -99,12 +102,16 @@ func slsaMaterials(srcs Sources) ([]slsa.ProvenanceMaterial, error) { }) } packageurl.NewPackageURL(packageurl.TypeOCI, "", s.Ref, "", q, "") - out = append(out, slsa.ProvenanceMaterial{ + + material := slsa.ProvenanceMaterial{ URI: s.Ref, - Digest: slsa.DigestSet{ + } + if s.Digest != "" { + material.Digest = slsa.DigestSet{ s.Digest.Algorithm().String(): s.Digest.Hex(), - }, - }) + } + } + out = append(out, material) } return out, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index e5e14460c3382..380f42fa0d493 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -576,7 +576,7 @@ github.com/mistifyio/go-zfs # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 -# github.com/moby/buildkit v0.11.6 +# github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f ## explicit; go 1.18 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types @@ -826,10 +826,6 @@ github.com/moby/term/windows # github.com/morikuni/aec v1.0.0 ## explicit github.com/morikuni/aec -# github.com/onsi/ginkgo/v2 v2.1.4 -## explicit; go 1.18 -# github.com/onsi/gomega v1.20.1 -## explicit; go 1.18 # github.com/opencontainers/go-digest v1.0.0 ## explicit; go 1.13 github.com/opencontainers/go-digest From 7a4ea198032957918ffb4359d1b621b8cfd82201 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Thu, 25 May 2023 16:00:29 -0400 Subject: [PATCH 043/293] libcontainerd: work around exec start bug in c8d It turns out that the unnecessary serialization removed in b75246202ab9b1e5bb94c377f90db8ed38cfa0e0 happened to work around a bug in containerd. When many exec processes are started concurrently in the same containerd task, it takes seconds to minutes for them all to start. Add the workaround back in, only deliberately this time. Signed-off-by: Cory Snider (cherry picked from commit fb7ec1555cca750f5d1d99bae5e99b4818712322) Signed-off-by: Sebastiaan van Stijn --- libcontainerd/remote/client.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libcontainerd/remote/client.go b/libcontainerd/remote/client.go index 1f96986159db9..837f770e3cd7b 100644 --- a/libcontainerd/remote/client.go +++ b/libcontainerd/remote/client.go @@ -60,6 +60,10 @@ type container struct { type task struct { containerd.Task ctr *container + + // Workaround for https://github.com/containerd/containerd/issues/8557. + // See also https://github.com/moby/moby/issues/45595. + serializeExecStartsWorkaround sync.Mutex } type process struct { @@ -296,7 +300,12 @@ func (t *task) Exec(ctx context.Context, processID string, spec *specs.Process, // the stdin of exec process will be created after p.Start in containerd defer func() { stdinCloseSync <- p }() - if err = p.Start(ctx); err != nil { + err = func() error { + t.serializeExecStartsWorkaround.Lock() + defer t.serializeExecStartsWorkaround.Unlock() + return p.Start(ctx) + }() + if err != nil { // use new context for cleanup because old one may be cancelled by user, but leave a timeout to make sure // we are not waiting forever if containerd is unresponsive or to work around fifo cancelling issues in // older containerd-shim From ec8ec9056cd9b3294eee98177acf18a57168197c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 26 May 2023 00:05:08 +0200 Subject: [PATCH 044/293] builder/remotecontext: deprecate CachableSource, NewCachableSource This type (as well as TarsumBackup), was used for the experimental --stream support for the classic builder. This feature was removed in commit 6ca3ec88ae9e1435abbed665ec598c00058659da, which also removed uses of the CachableSource type. As far as I could find, there's no external consumers of these types, but let's deprecated it, to give potential users a heads-up that it will be removed. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 37d4b0bee98c9c7cfe66516c0be0dda43080d188) Signed-off-by: Sebastiaan van Stijn --- builder/remotecontext/tarsum.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/builder/remotecontext/tarsum.go b/builder/remotecontext/tarsum.go index b2bd8d9c3f4f8..19e409eee501c 100644 --- a/builder/remotecontext/tarsum.go +++ b/builder/remotecontext/tarsum.go @@ -15,7 +15,9 @@ type hashed interface { Digest() digest.Digest } -// CachableSource is a source that contains cache records for its contents +// CachableSource is a source that contains cache records for its contents. +// +// Deprecated: this type was used for the experimental "stream" support for the classic builder, which is no longer supported. type CachableSource struct { mu sync.Mutex root string @@ -23,7 +25,9 @@ type CachableSource struct { txn *iradix.Txn } -// NewCachableSource creates new CachableSource +// NewCachableSource creates new CachableSource. +// +// Deprecated: this type was used for the experimental "stream" support for the classic builder, which is no longer supported. func NewCachableSource(root string) *CachableSource { ts := &CachableSource{ tree: iradix.New(), From 042f0799dbbc8293cd64ae48b46af05a7fad783c Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Fri, 26 May 2023 14:36:34 -0400 Subject: [PATCH 045/293] libn/d/overlay: support encryption on any port While the VXLAN interface and the iptables rules to mark outgoing VXLAN packets for encryption are configured to use the Swarm data path port, the XFRM policies for actually applying the encryption are hardcoded to match packets with destination port 4789/udp. Consequently, encrypted overlay networks do not pass traffic when the Swarm is configured with any other data path port: encryption is not applied to the outgoing VXLAN packets and the destination host drops the received cleartext packets. Use the configured data path port instead of hardcoding port 4789 in the XFRM policies. Signed-off-by: Cory Snider (cherry picked from commit 9a692a38028f4914a3a914c9a229e61bb3fbaf66) Signed-off-by: Cory Snider --- libnetwork/drivers/overlay/encryption.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libnetwork/drivers/overlay/encryption.go b/libnetwork/drivers/overlay/encryption.go index 79f7169b6d949..a9104d1820469 100644 --- a/libnetwork/drivers/overlay/encryption.go +++ b/libnetwork/drivers/overlay/encryption.go @@ -375,8 +375,8 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error { Src: &net.IPNet{IP: s, Mask: fullMask}, Dst: &net.IPNet{IP: d, Mask: fullMask}, Dir: netlink.XFRM_DIR_OUT, - Proto: 17, - DstPort: 4789, + Proto: syscall.IPPROTO_UDP, + DstPort: int(overlayutils.VXLANUDPPort()), Mark: &spMark, Tmpls: []netlink.XfrmPolicyTmpl{ { @@ -589,8 +589,8 @@ func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, pr Src: &net.IPNet{IP: s, Mask: fullMask}, Dst: &net.IPNet{IP: d, Mask: fullMask}, Dir: netlink.XFRM_DIR_OUT, - Proto: 17, - DstPort: 4789, + Proto: syscall.IPPROTO_UDP, + DstPort: int(overlayutils.VXLANUDPPort()), Mark: &spMark, Tmpls: []netlink.XfrmPolicyTmpl{ { From d9e39914a73fd6aead461fcc6d50bf499780c7a4 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Sat, 27 May 2023 16:04:59 +0000 Subject: [PATCH 046/293] Fix npe in exec resize when exec errored In cases where an exec start failed the exec process will be nil even though the channel to signal that the exec started was closed. Ideally ExecConfig would get a nice refactor to handle this case better (ie. it's not started so don't close that channel). This is a minimal fix to prevent NPE. Luckilly this would only get called by a client and only the http request goroutine gets the panic (http lib recovers the panic). Signed-off-by: Brian Goff (cherry picked from commit 487ea813163903df78e5449b9c9b1cc02cef6ae2) Signed-off-by: Sebastiaan van Stijn --- daemon/resize.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/daemon/resize.go b/daemon/resize.go index d1325e629784c..8e350fbbc2e7d 100644 --- a/daemon/resize.go +++ b/daemon/resize.go @@ -5,6 +5,8 @@ import ( "errors" "strconv" "time" + + "github.com/docker/docker/errdefs" ) // ContainerResize changes the size of the TTY of the process running @@ -48,6 +50,10 @@ func (daemon *Daemon) ContainerExecResize(name string, height, width int) error select { case <-ec.Started: + // An error may have occurred, so ec.Process may be nil. + if ec.Process == nil { + return errdefs.InvalidParameter(errors.New("exec process is not started")) + } return ec.Process.Resize(context.Background(), uint32(width), uint32(height)) case <-timeout.C: return errors.New("timeout waiting for exec session ready") From fec801a103566d232097d0479182158de7a1d42d Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Tue, 30 May 2023 12:08:12 -0400 Subject: [PATCH 047/293] libnetwork: log why osl sandbox restore failed Signed-off-by: Cory Snider (cherry picked from commit 18bf3aa4421e3c14fc5bed95e0e806b04879d8fa) Signed-off-by: Cory Snider --- libnetwork/sandbox_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libnetwork/sandbox_store.go b/libnetwork/sandbox_store.go index 92eaf7a1554ee..fc6b0071ff7f2 100644 --- a/libnetwork/sandbox_store.go +++ b/libnetwork/sandbox_store.go @@ -260,7 +260,7 @@ func (c *Controller) sandboxCleanup(activeSandboxes map[string]interface{}) { // reconstruct osl sandbox field if !sb.config.useDefaultSandBox { if err := sb.restoreOslSandbox(); err != nil { - logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID()) + logrus.Errorf("failed to populate fields for osl sandbox %s: %v", sb.ID(), err) continue } } else { From 3452a76589ca82300fe2fff2434362f0cb847bd3 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Tue, 30 May 2023 12:27:59 -0400 Subject: [PATCH 048/293] libnetwork: fix sandbox restore The method to restore a network namespace takes a collection of interfaces to restore with the options to apply. The interface names are structured data, tuples of (SrcName, DstPrefix) but for whatever reason are being passed into Restore() serialized to strings. A refactor, f0be4d126dacb19a35046536b041a5989ff0dc95, accidentally broke the serialization by dropping the delimiter. Rather than fix the serialization and leave the time-bomb for someone else to trip over, pass the interface names as structured data. Signed-off-by: Cory Snider (cherry picked from commit 50eb2d27821df9ec19218c325534ae42d63016aa) Signed-off-by: Cory Snider --- libnetwork/osl/namespace_linux.go | 16 +++++----------- libnetwork/osl/sandbox.go | 6 +++++- libnetwork/sandbox.go | 4 ++-- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/libnetwork/osl/namespace_linux.go b/libnetwork/osl/namespace_linux.go index 9f22d8077208b..d7d2fe2d63769 100644 --- a/libnetwork/osl/namespace_linux.go +++ b/libnetwork/osl/namespace_linux.go @@ -470,16 +470,10 @@ func (n *networkNamespace) Destroy() error { } // Restore restore the network namespace -func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error { +func (n *networkNamespace) Restore(ifsopt map[Iface][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error { // restore interfaces for name, opts := range ifsopt { - if !strings.Contains(name, "+") { - return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name) - } - seps := strings.Split(name, "+") - srcName := seps[0] - dstPrefix := seps[1] - i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n} + i := &nwIface{srcName: name.SrcName, dstName: name.DstPrefix, ns: n} i.processInterfaceOptions(opts...) if i.master != "" { i.dstMaster = n.findDst(i.master, true) @@ -531,7 +525,7 @@ func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*ty } var index int - indexStr := strings.TrimPrefix(i.dstName, dstPrefix) + indexStr := strings.TrimPrefix(i.dstName, name.DstPrefix) if indexStr != "" { index, err = strconv.Atoi(indexStr) if err != nil { @@ -540,8 +534,8 @@ func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*ty } index++ n.Lock() - if index > n.nextIfIndex[dstPrefix] { - n.nextIfIndex[dstPrefix] = index + if index > n.nextIfIndex[name.DstPrefix] { + n.nextIfIndex[name.DstPrefix] = index } n.iFaces = append(n.iFaces, i) n.Unlock() diff --git a/libnetwork/osl/sandbox.go b/libnetwork/osl/sandbox.go index 84833167e645e..9af7e46d234b6 100644 --- a/libnetwork/osl/sandbox.go +++ b/libnetwork/osl/sandbox.go @@ -17,6 +17,10 @@ const ( SandboxTypeLoadBalancer = iota ) +type Iface struct { + SrcName, DstPrefix string +} + // IfaceOption is a function option type to set interface options. type IfaceOption func(i *nwIface) @@ -89,7 +93,7 @@ type Sandbox interface { Destroy() error // Restore restores the sandbox. - Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error + Restore(ifsopt map[Iface][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error // ApplyOSTweaks applies operating system specific knobs on the sandbox. ApplyOSTweaks([]SandboxType) diff --git a/libnetwork/sandbox.go b/libnetwork/sandbox.go index 194844ca7b804..aceb7b9be5b81 100644 --- a/libnetwork/sandbox.go +++ b/libnetwork/sandbox.go @@ -765,7 +765,7 @@ func (sb *Sandbox) restoreOslSandbox() error { var routes []*types.StaticRoute // restore osl sandbox - Ifaces := make(map[string][]osl.IfaceOption) + Ifaces := make(map[osl.Iface][]osl.IfaceOption) for _, ep := range sb.endpoints { ep.mu.Lock() joinInfo := ep.joinInfo @@ -790,7 +790,7 @@ func (sb *Sandbox) restoreOslSandbox() error { if len(i.llAddrs) != 0 { ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) } - Ifaces[i.srcName+i.dstPrefix] = ifaceOptions + Ifaces[osl.Iface{SrcName: i.srcName, DstPrefix: i.dstPrefix}] = ifaceOptions if joinInfo != nil { routes = append(routes, joinInfo.StaticRoutes...) } From f9c68e5fbc8ab213e44728323d17be3e0291b43c Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Tue, 30 May 2023 14:32:27 -0400 Subject: [PATCH 049/293] libn: fix resolver restore w/ chatty 'iptables -C' Resolver.setupIPTable() checks whether it needs to flush or create the user chains used for NATing container DNS requests by testing for the existence of the rules which jump to said user chains. Unfortunately it does so using the IPTable.RawCombinedOutputNative() method, which returns a non-nil error if the iptables command returns any output even if the command exits with a zero status code. While that is fine with iptables-legacy as it prints no output if the rule exists, iptables-nft v1.8.7 prints some information about the rule. Consequently, Resolver.setupIPTable() would incorrectly think that the rule does not exist during container restore and attempt to create it. This happened work work by coincidence before 8f5a9a741b70852bc6c9d675c6e0c4944022b467 because the failure to create the already-existing table would be ignored and the new NAT rules would be inserted before the stale rules left in the table from when the container was last started/restored. Now that failing to create the table is treated as a fatal error, the incompatibility with iptables-nft is no longer hidden. Switch to using IPTable.ExistsNative() to test for the existence of the jump rules as it correctly only checks the iptables command's exit status without regard for whether it outputs anything. Signed-off-by: Cory Snider (cherry picked from commit 117831931394fcbb6e547ec0dcde10ad6ac135a9) Signed-off-by: Cory Snider --- libnetwork/resolver_unix.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libnetwork/resolver_unix.go b/libnetwork/resolver_unix.go index 7b0511bcff493..466a89db087ca 100644 --- a/libnetwork/resolver_unix.go +++ b/libnetwork/resolver_unix.go @@ -38,8 +38,7 @@ func (r *Resolver) setupIPTable() error { iptable := iptables.GetIptable(iptables.IPv4) // insert outputChain and postroutingchain - err := iptable.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain) - if err == nil { + if iptable.ExistsNative("nat", "OUTPUT", "-d", resolverIP, "-j", outputChain) { if err := iptable.RawCombinedOutputNative("-t", "nat", "-F", outputChain); err != nil { setupErr = err return @@ -55,8 +54,7 @@ func (r *Resolver) setupIPTable() error { } } - err = iptable.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingChain) - if err == nil { + if iptable.ExistsNative("nat", "POSTROUTING", "-d", resolverIP, "-j", postroutingChain) { if err := iptable.RawCombinedOutputNative("-t", "nat", "-F", postroutingChain); err != nil { setupErr = err return From 2949fee1d382086047cd0b8d04fc9844d768d9a2 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 1 Jun 2023 13:33:16 +0200 Subject: [PATCH 050/293] containerd: set user-agent when pushing/pulling images Before this, the client would report itself as containerd, and the containerd version from the containerd go module: time="2023-06-01T09:43:21.907359755Z" level=info msg="listening on [::]:5000" go.version=go1.19.9 instance.id=67b89d83-eac0-4f85-b36b-b1b18e80bde1 service=registry version=2.8.2 ... 172.18.0.1 - - [01/Jun/2023:09:43:33 +0000] "HEAD /v2/multifoo/blobs/sha256:cb269d7c0c1ca22fb5a70342c3ed2196c57a825f94b3f0e5ce3aa8c55baee829 HTTP/1.1" 404 157 "" "containerd/1.6.21+unknown" With this patch, the user-agent has the docker daemon information; time="2023-06-01T11:27:07.959822887Z" level=info msg="listening on [::]:5000" go.version=go1.19.9 instance.id=53590f34-096a-4fd1-9c58-d3b8eb7e5092 service=registry version=2.8.2 ... 172.18.0.1 - - [01/Jun/2023:11:27:20 +0000] "HEAD /v2/multifoo/blobs/sha256:c7ec7661263e5e597156f2281d97b160b91af56fa1fd2cc045061c7adac4babd HTTP/1.1" 404 157 "" "docker/dev go/go1.20.4 git-commit/8d67d0c1a8 kernel/5.15.49-linuxkit-pr os/linux arch/arm64 UpstreamClient(Docker-Client/24.0.2 \\(linux\\))" Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 66137ae42914c647e793a1f7ac566bf92649ccf2) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/image_pull.go | 2 +- daemon/containerd/image_push.go | 2 +- daemon/containerd/resolver.go | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/daemon/containerd/image_pull.go b/daemon/containerd/image_pull.go index 160d47cc2f2ac..450a03fb9fe02 100644 --- a/daemon/containerd/image_pull.go +++ b/daemon/containerd/image_pull.go @@ -45,7 +45,7 @@ func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string, } } - resolver, _ := i.newResolverFromAuthConfig(authConfig) + resolver, _ := i.newResolverFromAuthConfig(ctx, authConfig) opts = append(opts, containerd.WithResolver(resolver)) jobs := newJobs() diff --git a/daemon/containerd/image_push.go b/daemon/containerd/image_push.go index ae3814761d312..f0a38ef63bf4e 100644 --- a/daemon/containerd/image_push.go +++ b/daemon/containerd/image_push.go @@ -62,7 +62,7 @@ func (i *ImageService) PushImage(ctx context.Context, targetRef reference.Named, target := img.Target store := i.client.ContentStore() - resolver, tracker := i.newResolverFromAuthConfig(authConfig) + resolver, tracker := i.newResolverFromAuthConfig(ctx, authConfig) progress := pushProgress{Tracker: tracker} jobsQueue := newJobs() finishProgress := jobsQueue.showProgress(ctx, out, combinedProgress([]progressUpdater{ diff --git a/daemon/containerd/resolver.go b/daemon/containerd/resolver.go index 5edd83181b19f..97bc9780df31b 100644 --- a/daemon/containerd/resolver.go +++ b/daemon/containerd/resolver.go @@ -1,6 +1,7 @@ package containerd import ( + "context" "crypto/tls" "errors" "net/http" @@ -8,19 +9,23 @@ import ( "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" registrytypes "github.com/docker/docker/api/types/registry" + "github.com/docker/docker/dockerversion" "github.com/docker/docker/registry" "github.com/sirupsen/logrus" ) -func (i *ImageService) newResolverFromAuthConfig(authConfig *registrytypes.AuthConfig) (remotes.Resolver, docker.StatusTracker) { +func (i *ImageService) newResolverFromAuthConfig(ctx context.Context, authConfig *registrytypes.AuthConfig) (remotes.Resolver, docker.StatusTracker) { tracker := docker.NewInMemoryTracker() hostsFn := i.registryHosts.RegistryHosts() hosts := hostsWrapper(hostsFn, authConfig, i.registryService) + headers := http.Header{} + headers.Set("User-Agent", dockerversion.DockerUserAgent(ctx)) return docker.NewResolver(docker.ResolverOptions{ Hosts: hosts, Tracker: tracker, + Headers: headers, }), tracker } From ac1a867282c6fb1619e41e8931e4b9b19a6d8164 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 23 May 2023 12:52:16 +0200 Subject: [PATCH 051/293] vendor: github.com/mistifyio/go-zfs/v3 v3.0.1 Switching to the v3 version, which was renamed to be compatible with go modules. Full diff: https://github.com/mistifyio/go-zfs/compare/f784269be439...v3.0.1 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 86108812b8e4372168bc8274dd1290cd3ef46de5) Signed-off-by: Sebastiaan van Stijn --- daemon/graphdriver/zfs/zfs.go | 2 +- vendor.mod | 2 +- vendor.sum | 3 +- vendor/github.com/mistifyio/go-zfs/.gitignore | 1 - .../github.com/mistifyio/go-zfs/.travis.yml | 43 --- .../github.com/mistifyio/go-zfs/Vagrantfile | 34 --- .../mistifyio/go-zfs/utils_notsolaris.go | 17 -- .../mistifyio/go-zfs/utils_solaris.go | 17 -- .../github.com/mistifyio/go-zfs/v3/.gitignore | 6 + .../mistifyio/go-zfs/v3/.golangci.yml | 207 +++++++++++++++ .../github.com/mistifyio/go-zfs/v3/.yamllint | 16 ++ .../mistifyio/go-zfs/v3/CHANGELOG.md | 250 ++++++++++++++++++ .../mistifyio/go-zfs/{ => v3}/CONTRIBUTING.md | 44 +-- .../mistifyio/go-zfs/{ => v3}/LICENSE | 0 .../github.com/mistifyio/go-zfs/v3/Makefile | 19 ++ .../mistifyio/go-zfs/{ => v3}/README.md | 13 +- .../mistifyio/go-zfs/v3/Vagrantfile | 33 +++ .../mistifyio/go-zfs/{ => v3}/error.go | 0 vendor/github.com/mistifyio/go-zfs/v3/lint.mk | 75 ++++++ .../github.com/mistifyio/go-zfs/v3/rules.mk | 49 ++++ .../github.com/mistifyio/go-zfs/v3/shell.nix | 26 ++ .../mistifyio/go-zfs/{ => v3}/utils.go | 99 ++++--- .../mistifyio/go-zfs/v3/utils_notsolaris.go | 19 ++ .../mistifyio/go-zfs/v3/utils_solaris.go | 19 ++ .../mistifyio/go-zfs/{ => v3}/zfs.go | 165 ++++++------ .../mistifyio/go-zfs/{ => v3}/zpool.go | 40 +-- vendor/modules.txt | 6 +- 27 files changed, 907 insertions(+), 298 deletions(-) delete mode 100644 vendor/github.com/mistifyio/go-zfs/.gitignore delete mode 100644 vendor/github.com/mistifyio/go-zfs/.travis.yml delete mode 100644 vendor/github.com/mistifyio/go-zfs/Vagrantfile delete mode 100644 vendor/github.com/mistifyio/go-zfs/utils_notsolaris.go delete mode 100644 vendor/github.com/mistifyio/go-zfs/utils_solaris.go create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/.gitignore create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/.golangci.yml create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/.yamllint create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/CHANGELOG.md rename vendor/github.com/mistifyio/go-zfs/{ => v3}/CONTRIBUTING.md (54%) rename vendor/github.com/mistifyio/go-zfs/{ => v3}/LICENSE (100%) create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/Makefile rename vendor/github.com/mistifyio/go-zfs/{ => v3}/README.md (87%) create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/Vagrantfile rename vendor/github.com/mistifyio/go-zfs/{ => v3}/error.go (100%) create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/lint.mk create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/rules.mk create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/shell.nix rename vendor/github.com/mistifyio/go-zfs/{ => v3}/utils.go (72%) create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/utils_notsolaris.go create mode 100644 vendor/github.com/mistifyio/go-zfs/v3/utils_solaris.go rename vendor/github.com/mistifyio/go-zfs/{ => v3}/zfs.go (70%) rename vendor/github.com/mistifyio/go-zfs/{ => v3}/zpool.go (70%) diff --git a/daemon/graphdriver/zfs/zfs.go b/daemon/graphdriver/zfs/zfs.go index b704a7905251f..b00959428702a 100644 --- a/daemon/graphdriver/zfs/zfs.go +++ b/daemon/graphdriver/zfs/zfs.go @@ -16,7 +16,7 @@ import ( "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/parsers" - zfs "github.com/mistifyio/go-zfs" + zfs "github.com/mistifyio/go-zfs/v3" "github.com/moby/locker" "github.com/moby/sys/mount" "github.com/moby/sys/mountinfo" diff --git a/vendor.mod b/vendor.mod index 12568d6b68e21..7f1c45b67e501 100644 --- a/vendor.mod +++ b/vendor.mod @@ -55,7 +55,7 @@ require ( github.com/ishidawataru/sctp v0.0.0-20230406120618-7ff4192f6ff2 github.com/klauspost/compress v1.16.3 github.com/miekg/dns v1.1.43 - github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible + github.com/mistifyio/go-zfs/v3 v3.0.1 github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go github.com/moby/ipvs v1.1.0 github.com/moby/locker v1.0.1 diff --git a/vendor.sum b/vendor.sum index 985ad6d0f59a4..8214f95e95a2f 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1024,8 +1024,9 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mistifyio/go-zfs/v3 v3.0.1 h1:YaoXgBePoMA12+S1u/ddkv+QqxcfiZK4prI6HPnkFiU= +github.com/mistifyio/go-zfs/v3 v3.0.1/go.mod h1:CzVgeB0RvF2EGzQnytKVvVSDwmKJXxkOTUGbNrTja/k= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= diff --git a/vendor/github.com/mistifyio/go-zfs/.gitignore b/vendor/github.com/mistifyio/go-zfs/.gitignore deleted file mode 100644 index 8000dd9db47c0..0000000000000 --- a/vendor/github.com/mistifyio/go-zfs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.vagrant diff --git a/vendor/github.com/mistifyio/go-zfs/.travis.yml b/vendor/github.com/mistifyio/go-zfs/.travis.yml deleted file mode 100644 index acbd39cefe928..0000000000000 --- a/vendor/github.com/mistifyio/go-zfs/.travis.yml +++ /dev/null @@ -1,43 +0,0 @@ -language: go -dist: trusty -sudo: required -cache: - directories: - - $HOME/.ccache - - $HOME/zfs - -branches: - only: - - master - -env: - - rel=0.6.5.11 - - rel=0.7.6 - -go: - - "1.10.x" - - master - -before_install: - - export MAKEFLAGS=-j$(($(grep -c '^processor' /proc/cpuinfo) * 2 + 1)) - - export PATH=/usr/lib/ccache:$PATH - - go get github.com/alecthomas/gometalinter - - gometalinter --install --update - - sudo apt-get update -y && sudo apt-get install -y libattr1-dev libblkid-dev linux-headers-$(uname -r) tree uuid-dev - - mkdir -p $HOME/zfs - - cd $HOME/zfs - - [[ -d spl-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/spl-$rel.tar.gz | tar xz - - [[ -d zfs-$rel.tar.gz ]] || curl -L https://github.com/zfsonlinux/zfs/releases/download/zfs-$rel/zfs-$rel.tar.gz | tar xz - - (cd spl-$rel && ./configure --prefix=/usr && make && sudo make install) - - (cd zfs-$rel && ./configure --prefix=/usr && make && sudo make install) - - sudo modprobe zfs - - cd $TRAVIS_BUILD_DIR - -script: - - sudo -E $(which go) test -v ./... - - gometalinter --vendor --vendored-linters ./... || true - - gometalinter --errors --vendor --vendored-linters ./... - -notifications: - email: false - irc: "chat.freenode.net#cerana" diff --git a/vendor/github.com/mistifyio/go-zfs/Vagrantfile b/vendor/github.com/mistifyio/go-zfs/Vagrantfile deleted file mode 100644 index 3bd6e120bc4af..0000000000000 --- a/vendor/github.com/mistifyio/go-zfs/Vagrantfile +++ /dev/null @@ -1,34 +0,0 @@ - -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/trusty64" - config.ssh.forward_agent = true - - config.vm.synced_folder ".", "/home/vagrant/go/src/github.com/mistifyio/go-zfs", create: true - - config.vm.provision "shell", inline: < /etc/profile.d/go.sh -export GOPATH=\\$HOME/go -export PATH=\\$GOPATH/bin:/usr/local/go/bin:\\$PATH -END - -chown -R vagrant /home/vagrant/go - -apt-get update -apt-get install -y software-properties-common curl -apt-add-repository --yes ppa:zfs-native/stable -apt-get update -apt-get install -y ubuntu-zfs - -cd /home/vagrant -curl -z go1.3.3.linux-amd64.tar.gz -L -O https://storage.googleapis.com/golang/go1.3.3.linux-amd64.tar.gz -tar -C /usr/local -zxf /home/vagrant/go1.3.3.linux-amd64.tar.gz - -cat << END > /etc/sudoers.d/go -Defaults env_keep += "GOPATH" -END - -EOF - -end diff --git a/vendor/github.com/mistifyio/go-zfs/utils_notsolaris.go b/vendor/github.com/mistifyio/go-zfs/utils_notsolaris.go deleted file mode 100644 index a46f73060d177..0000000000000 --- a/vendor/github.com/mistifyio/go-zfs/utils_notsolaris.go +++ /dev/null @@ -1,17 +0,0 @@ -// +build !solaris - -package zfs - -import ( - "strings" -) - -// List of ZFS properties to retrieve from zfs list command on a non-Solaris platform -var dsPropList = []string{"name", "origin", "used", "available", "mountpoint", "compression", "type", "volsize", "quota", "referenced", "written", "logicalused", "usedbydataset"} - -var dsPropListOptions = strings.Join(dsPropList, ",") - -// List of Zpool properties to retrieve from zpool list command on a non-Solaris platform -var zpoolPropList = []string{"name", "health", "allocated", "size", "free", "readonly", "dedupratio", "fragmentation", "freeing", "leaked"} -var zpoolPropListOptions = strings.Join(zpoolPropList, ",") -var zpoolArgs = []string{"get", "-p", zpoolPropListOptions} diff --git a/vendor/github.com/mistifyio/go-zfs/utils_solaris.go b/vendor/github.com/mistifyio/go-zfs/utils_solaris.go deleted file mode 100644 index 0a7e90f22276f..0000000000000 --- a/vendor/github.com/mistifyio/go-zfs/utils_solaris.go +++ /dev/null @@ -1,17 +0,0 @@ -// +build solaris - -package zfs - -import ( - "strings" -) - -// List of ZFS properties to retrieve from zfs list command on a Solaris platform -var dsPropList = []string{"name", "origin", "used", "available", "mountpoint", "compression", "type", "volsize", "quota", "referenced"} - -var dsPropListOptions = strings.Join(dsPropList, ",") - -// List of Zpool properties to retrieve from zpool list command on a non-Solaris platform -var zpoolPropList = []string{"name", "health", "allocated", "size", "free", "readonly", "dedupratio"} -var zpoolPropListOptions = strings.Join(zpoolPropList, ",") -var zpoolArgs = []string{"get", "-p", zpoolPropListOptions} diff --git a/vendor/github.com/mistifyio/go-zfs/v3/.gitignore b/vendor/github.com/mistifyio/go-zfs/v3/.gitignore new file mode 100644 index 0000000000000..0867490ad59c4 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/.gitignore @@ -0,0 +1,6 @@ +bin +go-zfs.test +.vagrant + +# added by lint-install +out/ diff --git a/vendor/github.com/mistifyio/go-zfs/v3/.golangci.yml b/vendor/github.com/mistifyio/go-zfs/v3/.golangci.yml new file mode 100644 index 0000000000000..499c3eca16668 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/.golangci.yml @@ -0,0 +1,207 @@ +run: + # The default runtime timeout is 1m, which doesn't work well on Github Actions. + timeout: 4m + +# NOTE: This file is populated by the lint-install tool. Local adjustments may be overwritten. +linters-settings: + cyclop: + # NOTE: This is a very high transitional threshold + max-complexity: 37 + package-average: 34.0 + skip-tests: true + + gocognit: + # NOTE: This is a very high transitional threshold + min-complexity: 98 + + dupl: + threshold: 200 + + goconst: + min-len: 4 + min-occurrences: 5 + ignore-tests: true + + gosec: + excludes: + - G107 # Potential HTTP request made with variable url + - G204 # Subprocess launched with function call as argument or cmd arguments + - G404 # Use of weak random number generator (math/rand instead of crypto/rand + + errorlint: + # these are still common in Go: for instance, exit errors. + asserts: false + + exhaustive: + default-signifies-exhaustive: true + + nestif: + min-complexity: 8 + + nolintlint: + require-explanation: true + allow-unused: false + require-specific: true + + revive: + ignore-generated-header: true + severity: warning + rules: + - name: atomic + - name: blank-imports + - name: bool-literal-in-expr + - name: confusing-naming + - name: constant-logical-expr + - name: context-as-argument + - name: context-keys-type + - name: deep-exit + - name: defer + - name: range-val-in-closure + - name: range-val-address + - name: dot-imports + - name: error-naming + - name: error-return + - name: error-strings + - name: errorf + - name: exported + - name: identical-branches + - name: if-return + - name: import-shadowing + - name: increment-decrement + - name: indent-error-flow + - name: indent-error-flow + - name: package-comments + - name: range + - name: receiver-naming + - name: redefines-builtin-id + - name: superfluous-else + - name: struct-tag + - name: time-naming + - name: unexported-naming + - name: unexported-return + - name: unnecessary-stmt + - name: unreachable-code + - name: unused-parameter + - name: var-declaration + - name: var-naming + - name: unconditional-recursion + - name: waitgroup-by-value + + staticcheck: + go: "1.16" + + unused: + go: "1.16" + +output: + sort-results: true + +linters: + disable-all: true + enable: + - asciicheck + - bodyclose + - cyclop + - deadcode + - dogsled + - dupl + - durationcheck + - errcheck + - errname + - errorlint + - exhaustive + - exportloopref + - forcetypeassert + - gocognit + - goconst + - gocritic + - godot + - gofmt + - gofumpt + - gosec + - goheader + - goimports + - goprintffuncname + - gosimple + - govet + - ifshort + - importas + - ineffassign + - makezero + - misspell + - nakedret + - nestif + - nilerr + - noctx + - nolintlint + - predeclared + # disabling for the initial iteration of the linting tool + # - promlinter + - revive + - rowserrcheck + - sqlclosecheck + - staticcheck + - structcheck + - stylecheck + - thelper + - tparallel + - typecheck + - unconvert + - unparam + - unused + - varcheck + - wastedassign + - whitespace + + # Disabled linters, due to being misaligned with Go practices + # - exhaustivestruct + # - gochecknoglobals + # - gochecknoinits + # - goconst + # - godox + # - goerr113 + # - gomnd + # - lll + # - nlreturn + # - testpackage + # - wsl + # Disabled linters, due to not being relevant to our code base: + # - maligned + # - prealloc "For most programs usage of prealloc will be a premature optimization." + # Disabled linters due to bad error messages or bugs + # - tagliatelle + +issues: + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + - path: _test\.go + linters: + - dupl + - errcheck + - forcetypeassert + - gocyclo + - gosec + - noctx + + - path: .*cmd.* + linters: + - noctx + + - path: main\.go + linters: + - noctx + + - path: .*cmd.* + text: "deep-exit" + + - path: main\.go + text: "deep-exit" + + # This check is of questionable value + - linters: + - tparallel + text: "call t.Parallel on the top level as well as its subtests" + + # Don't hide lint issues just because there are many of them + max-same-issues: 0 + max-issues-per-linter: 0 diff --git a/vendor/github.com/mistifyio/go-zfs/v3/.yamllint b/vendor/github.com/mistifyio/go-zfs/v3/.yamllint new file mode 100644 index 0000000000000..9a08ad1765b99 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/.yamllint @@ -0,0 +1,16 @@ +--- +extends: default + +rules: + braces: + max-spaces-inside: 1 + brackets: + max-spaces-inside: 1 + comments: disable + comments-indentation: disable + document-start: disable + line-length: + level: warning + max: 160 + allow-non-breakable-inline-mappings: true + truthy: disable diff --git a/vendor/github.com/mistifyio/go-zfs/v3/CHANGELOG.md b/vendor/github.com/mistifyio/go-zfs/v3/CHANGELOG.md new file mode 100644 index 0000000000000..349245d039291 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/CHANGELOG.md @@ -0,0 +1,250 @@ +# Change Log + +All notable changes to this project will be documented in this file. +This project adheres to [Semantic Versioning](http://semver.org/). +This change log follows the advice of [Keep a CHANGELOG](https://github.com/olivierlacan/keep-a-changelog). + +## [Unreleased] + +## [3.0.0] - 2022-03-30 + +### Added + +- Rename, Mount and Unmount methods +- Parse more fields into Zpool type: + - dedupratio + - fragmentation + - freeing + - leaked + - readonly +- Parse more fields into Dataset type: + - referenced +- Incremental Send +- Parse numbers in exact format +- Support for Solaris (non-blockint, best-effort status) +- Debug logging for command invocation +- Use GitHub Actions for CI +- Nix shell for dev env reproducibility +- Direnv file for ease of dev +- Formatting/lint checks (enforced by CI) +- Go Module +- FreeBSD based vagrant machine + +### Changed + +- Temporarily adjust TestDiff expected strings depending on ZFS version +- Use one `zfs list`/`zpool list` call instead of many `zfs get`/`zpool get` +- ZFS docs links now point to OpenZFS pages +- Ubuntu vagrant box changed to generic/ubuntu2004 + +### Fixed + +- `GetProperty` returning `VALUE` instead of the actual value + +### Shortlog + + Amit Krishnan (1): + Issue #39 and Issue #40 - Enable Solaris support for go-zfs Switch from zfs/zpool get to zfs/zpool list for better performance Signed-off-by: Amit Krishnan + + Anand Patil (3): + Added Rename + Small fix to rename. + Added mount and umount methods + + Brian Akins (1): + Add 'referenced' to zfs properties + + Brian Bickerton (3): + Add debug logging before and after running external zfs command + Don't export the default no-op logger + Update uuid package repo url + + Dmitry Teselkin (1): + Issue #52 - fix parseLine for fragmentation field + + Edward Betts (1): + correct spelling mistake + + Justin Cormack (1): + Switch to google/uuid which is the maintained version of pborman/uuid + + Manuel Mendez (40): + rename Umount -> Unmount to follow zfs command name + add missing Unmount/Mount docs + always allocate largest Mount slice + add travis config + travis: update to go 1.7 + travis: get go deps first + test: add nok helper to verify an error occurred + test: add test for Dataset.GetProperty + ci: swap #cerana on freenode for slack + ci: install new deps for 0.7 relases + ci: bump zol versions + ci: bump go versions + ci: use better gometalinter invocations + ci: add ccache + ci: set env earlier in before_install + fix test nok error printing + test: restructure TestDiff to deal with different order of changes + test: better unicode path handling in TestDiff + travis: bump zfs and go versions + cache zfs artifacts + Add nix-shell and direnv goodness + prettierify all the files + Add go based tools + Add Makefile and rules.mk files + gofumptize the code base + Use tinkerbell/lint-install to setup linters + make golangci-lint happy + Update CONTRIBUTING.md with make based approach + Add GitHub Actions + Drop Travis CI + One sentence per line + Update documentation links to openzfs-docs pages + Format Vagrantfile using rufo + Add go-zfs.test to .gitignore + test: Avoid reptitive/duplicate error logging and quitting + test: Use t.Logf instead of fmt.Printf + test: Better cleanup and error handling in zpoolTest + test: Do not mark TestDatasets as a t.Helper. + test: Change zpoolTest to a pure helper that returns a clean up function + test: Move helpers to a different file + vagrant: Add set -euxo pipefail to provision script + vagrant: Update to generic/ubuntu2004 + vagrant: Minor fixes to Vagrantfile + vagrant: Update to go 1.17.8 + vagrant: Run go tests as part of provision script + vagrant: Indent heredoc script + vagrant: Add freebsd machine + + Matt Layher (1): + Parse more fields into Zpool type + + Michael Crosby (1): + Add incremental send + + Rikard Gynnerstedt (1): + remove command name from joined args + + Sebastiaan van Stijn (1): + Add go.mod and rename to github.com/mistifyio/go-zfs/v3 (v3.0.0) + + mikudeko (1): + Fix GetProperty always returning 'VALUE' + +## [2.1.1] - 2015-05-29 + +### Fixed + +- Ignoring first pool listed +- Incorrect `zfs get` argument ordering + +### Shortlog + + Alexey Guskov (1): + zfs command uses different order of arguments on freebsd + + Brian Akins (4): + test that ListZpools returns expected zpool + test error first + test error first + fix test to check correct return value + + James Cunningham (1): + Fix Truncating First Zpool + + Pat Norton (2): + Added Use of Go Tools + Update CONTRIBUTING.md + +## [2.1.0] - 2014-12-08 + +### Added + +- Parse hardlink modification count returned from `zfs diff` + +### Fixed + +- Continuing instead of erroring when rolling back a non-snapshot + +### Shortlog + + Brian Akins (2): + need to return the error here + use named struct fields + + Jörg Thalheim (1): + zfs diff handle hardlinks modification now + +## [2.0.0] - 2014-12-02 + +### Added + +- Flags for Destroy: + - DESTROY_DEFAULT + - DESTROY_DEFER_DELETION (`zfs destroy ... -d`) + - DESTROY_FORCE (`zfs destroy ... -f`) + - DESTROY_RECURSIVE_CLONES (`zfs destroy ... -R`) + - DESTROY_RECURSIVE (`zfs destroy ... -r`) + - etc +- Diff method (`zfs diff`) +- LogicalUsed and Origin properties to Dataset +- Type constants for Dataset +- State constants for Zpool +- Logger interface +- Improve documentation + +### Shortlog + + Brian Akins (8): + remove reflection + style change for switches + need to check for error + keep in scope + go 1.3.3 + golint cleanup + Just test if logical used is greater than 0, as this appears to be implementation specific + add docs to satisfy golint + + Jörg Thalheim (8): + Add deferred flag to zfs.Destroy() + add Logicalused property + Add Origin property + gofmt + Add zfs.Diff + Add Logger + add recursive destroy with clones + use CamelCase-style constants + + Matt Layher (4): + Improve documentation, document common ZFS operations, provide more references + Add zpool state constants, for easier health checking + Add dataset type constants, for easier type checking + Fix string split in command.Run(), use strings.Fields() instead of strings.Split() + +## [1.0.0] - 2014-11-12 + +### Shortlog + + Brian Akins (7): + add godoc badge + Add example + add information about zpool to struct and parser + Add Quota + add Children call + add Children call + fix snapshot tests + + Brian Bickerton (3): + MIST-150 Change Snapshot second paramater from properties map[string][string] to recursive bool + MIST-150 Add Rollback method and related tests + MIST-160 Add SendSnapshot streaming method and tests + + Matt Layher (1): + Add Error struct type and tests, enabling easier error return checking + +[3.0.0]: https://github.com/mistifyio/go-zfs/compare/v2.1.1...v3.0.0 +[2.1.1]: https://github.com/mistifyio/go-zfs/compare/v2.1.0...v2.1.1 +[2.1.0]: https://github.com/mistifyio/go-zfs/compare/v2.0.0...v2.1.0 +[2.0.0]: https://github.com/mistifyio/go-zfs/compare/v1.0.0...v2.0.0 +[1.0.0]: https://github.com/mistifyio/go-zfs/compare/v0.0.0...v1.0.0 diff --git a/vendor/github.com/mistifyio/go-zfs/CONTRIBUTING.md b/vendor/github.com/mistifyio/go-zfs/v3/CONTRIBUTING.md similarity index 54% rename from vendor/github.com/mistifyio/go-zfs/CONTRIBUTING.md rename to vendor/github.com/mistifyio/go-zfs/v3/CONTRIBUTING.md index f1880c19e544a..9f625d5646b65 100644 --- a/vendor/github.com/mistifyio/go-zfs/CONTRIBUTING.md +++ b/vendor/github.com/mistifyio/go-zfs/v3/CONTRIBUTING.md @@ -1,20 +1,23 @@ -## How to Contribute ## +## How to Contribute -We always welcome contributions to help make `go-zfs` better. Please take a moment to read this document if you would like to contribute. +We always welcome contributions to help make `go-zfs` better. +Please take a moment to read this document if you would like to contribute. -### Reporting issues ### +### Reporting issues We use [Github issues](https://github.com/mistifyio/go-zfs/issues) to track bug reports, feature requests, and submitting pull requests. If you find a bug: -* Use the GitHub issue search to check whether the bug has already been reported. -* If the issue has been fixed, try to reproduce the issue using the latest `master` branch of the repository. -* If the issue still reproduces or has not yet been reported, try to isolate the problem before opening an issue, if possible. Also provide the steps taken to reproduce the bug. +- Use the GitHub issue search to check whether the bug has already been reported. +- If the issue has been fixed, try to reproduce the issue using the latest `master` branch of the repository. +- If the issue still reproduces or has not yet been reported, try to isolate the problem before opening an issue, if possible. Also provide the steps taken to reproduce the bug. -### Pull requests ### +### Pull requests -We welcome bug fixes, improvements, and new features. Before embarking on making significant changes, please open an issue and ask first so that you do not risk duplicating efforts or spending time working on something that may be out of scope. For minor items, just open a pull request. +We welcome bug fixes, improvements, and new features. +Before embarking on making significant changes, please open an issue and ask first so that you do not risk duplicating efforts or spending time working on something that may be out of scope. +For minor items, just open a pull request. [Fork the project](https://help.github.com/articles/fork-a-repo), clone your fork, and add the upstream to your remote: @@ -28,11 +31,13 @@ If you need to pull new changes committed upstream: $ git fetch upstream $ git merge upstream/master -Don' work directly on master as this makes it harder to merge later. Create a feature branch for your fix or new feature: +Don' work directly on master as this makes it harder to merge later. +Create a feature branch for your fix or new feature: $ git checkout -b -Please try to commit your changes in logical chunks. Ideally, you should include the issue number in the commit message. +Please try to commit your changes in logical chunks. +Ideally, you should include the issue number in the commit message. $ git commit -m "Issue # - " @@ -40,21 +45,20 @@ Push your feature branch to your fork. $ git push origin -[Open a Pull Request](https://help.github.com/articles/using-pull-requests) against the upstream master branch. Please give your pull request a clear title and description and note which issue(s) your pull request fixes. +[Open a Pull Request](https://help.github.com/articles/using-pull-requests) against the upstream master branch. +Please give your pull request a clear title and description and note which issue(s) your pull request fixes. -* All Go code should be formatted using [gofmt](http://golang.org/cmd/gofmt/). -* Every exported function should have [documentation](http://blog.golang.org/godoc-documenting-go-code) and corresponding [tests](http://golang.org/doc/code.html#Testing). +- All linters should be happy (can be run with `make verify`). +- Every exported function should have [documentation](http://blog.golang.org/godoc-documenting-go-code) and corresponding [tests](http://golang.org/doc/code.html#Testing). **Important:** By submitting a patch, you agree to allow the project owners to license your work under the [Apache 2.0 License](./LICENSE). -### Go Tools ### -For consistency and to catch minor issues for all of go code, please run the following: -* goimports -* go vet -* golint -* errcheck +### Go Tools + +For consistency and to catch minor issues for all of go code, please run `make verify`. Many editors can execute the above on save. ----- +--- + Guidelines based on http://azkaban.github.io/contributing.html diff --git a/vendor/github.com/mistifyio/go-zfs/LICENSE b/vendor/github.com/mistifyio/go-zfs/v3/LICENSE similarity index 100% rename from vendor/github.com/mistifyio/go-zfs/LICENSE rename to vendor/github.com/mistifyio/go-zfs/v3/LICENSE diff --git a/vendor/github.com/mistifyio/go-zfs/v3/Makefile b/vendor/github.com/mistifyio/go-zfs/v3/Makefile new file mode 100644 index 0000000000000..1c5f55e8c6052 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/Makefile @@ -0,0 +1,19 @@ +help: ## Print this help + @grep --no-filename -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sed 's/:.*## /·/' | sort | column -t -W 2 -s '·' -c $(shell tput cols) + +all: test ## Run tests + +-include rules.mk +-include lint.mk + +test: ## Run tests + go test ./... + +verify: gofumpt prettier lint ## Verify code style, is lint free, freshness ... + git diff | (! grep .) + +fix: gofumpt-fix prettier-fix ## Fix code formatting errors + +tools: ${toolsBins} ## Build Go based build tools + +.PHONY: all help test tools verify diff --git a/vendor/github.com/mistifyio/go-zfs/README.md b/vendor/github.com/mistifyio/go-zfs/v3/README.md similarity index 87% rename from vendor/github.com/mistifyio/go-zfs/README.md rename to vendor/github.com/mistifyio/go-zfs/v3/README.md index fef80d727b18f..c911833002b33 100644 --- a/vendor/github.com/mistifyio/go-zfs/README.md +++ b/vendor/github.com/mistifyio/go-zfs/v3/README.md @@ -1,12 +1,12 @@ -# Go Wrapper for ZFS # +# Go Wrapper for ZFS Simple wrappers for ZFS command line tools. [![GoDoc](https://godoc.org/github.com/mistifyio/go-zfs?status.svg)](https://godoc.org/github.com/mistifyio/go-zfs) -## Requirements ## +## Requirements -You need a working ZFS setup. To use on Ubuntu 14.04, setup ZFS: +You need a working ZFS setup. To use on Ubuntu 14.04, setup ZFS: sudo apt-get install python-software-properties sudo apt-add-repository ppa:zfs-native/stable @@ -17,13 +17,13 @@ Developed using Go 1.3, but currently there isn't anything 1.3 specific. Don't u Generally you need root privileges to use anything zfs related. -## Status ## +## Status This has been only been tested on Ubuntu 14.04 In the future, we hope to work directly with libzfs. -# Hacking # +# Hacking The tests have decent examples for most functions. @@ -48,7 +48,6 @@ err := f.Destroy() ``` -# Contributing # +# Contributing See the [contributing guidelines](./CONTRIBUTING.md) - diff --git a/vendor/github.com/mistifyio/go-zfs/v3/Vagrantfile b/vendor/github.com/mistifyio/go-zfs/v3/Vagrantfile new file mode 100644 index 0000000000000..7d8d2decd3c40 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/Vagrantfile @@ -0,0 +1,33 @@ +GOVERSION = "1.17.8" + +Vagrant.configure("2") do |config| + config.vm.define "ubuntu" do |ubuntu| + ubuntu.vm.box = "generic/ubuntu2004" + end + config.vm.define "freebsd" do |freebsd| + freebsd.vm.box = "generic/freebsd13" + end + config.ssh.forward_agent = true + config.vm.synced_folder ".", "/home/vagrant/go/src/github.com/mistifyio/go-zfs", create: true + config.vm.provision "shell", inline: <<-EOF + set -euxo pipefail + + os=$(uname -s|tr '[A-Z]' '[a-z]') + case $os in + linux) apt-get update -y && apt-get install -y --no-install-recommends gcc libc-dev zfsutils-linux ;; + esac + + cd /tmp + curl -fLO --retry-max-time 30 --retry 10 https://go.dev/dl/go#{GOVERSION}.$os-amd64.tar.gz + tar -C /usr/local -zxf go#{GOVERSION}.$os-amd64.tar.gz + ln -nsf /usr/local/go/bin/go /usr/local/bin/go + rm -rf go*.tar.gz + + chown -R vagrant:vagrant /home/vagrant/go + cd /home/vagrant/go/src/github.com/mistifyio/go-zfs + go test -c + sudo ./go-zfs.test -test.v + CGO_ENABLED=0 go test -c + sudo ./go-zfs.test -test.v + EOF +end diff --git a/vendor/github.com/mistifyio/go-zfs/error.go b/vendor/github.com/mistifyio/go-zfs/v3/error.go similarity index 100% rename from vendor/github.com/mistifyio/go-zfs/error.go rename to vendor/github.com/mistifyio/go-zfs/v3/error.go diff --git a/vendor/github.com/mistifyio/go-zfs/v3/lint.mk b/vendor/github.com/mistifyio/go-zfs/v3/lint.mk new file mode 100644 index 0000000000000..a1e0a4fd36fc8 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/lint.mk @@ -0,0 +1,75 @@ +# BEGIN: lint-install -makefile lint.mk . +# http://github.com/tinkerbell/lint-install + +.PHONY: lint +lint: _lint + +LINT_ARCH := $(shell uname -m) +LINT_OS := $(shell uname) +LINT_OS_LOWER := $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]') +LINT_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) + +# shellcheck and hadolint lack arm64 native binaries: rely on x86-64 emulation +ifeq ($(LINT_OS),Darwin) + ifeq ($(LINT_ARCH),arm64) + LINT_ARCH=x86_64 + endif +endif + +LINTERS := +FIXERS := + +SHELLCHECK_VERSION ?= v0.8.0 +SHELLCHECK_BIN := out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH) +$(SHELLCHECK_BIN): + mkdir -p out/linters + rm -rf out/linters/shellcheck-* + curl -sSfL https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/shellcheck-$(SHELLCHECK_VERSION).$(LINT_OS_LOWER).$(LINT_ARCH).tar.xz | tar -C out/linters -xJf - + mv out/linters/shellcheck-$(SHELLCHECK_VERSION)/shellcheck $@ + rm -rf out/linters/shellcheck-$(SHELLCHECK_VERSION)/shellcheck + +LINTERS += shellcheck-lint +shellcheck-lint: $(SHELLCHECK_BIN) + $(SHELLCHECK_BIN) $(shell find . -name "*.sh") + +FIXERS += shellcheck-fix +shellcheck-fix: $(SHELLCHECK_BIN) + $(SHELLCHECK_BIN) $(shell find . -name "*.sh") -f diff | { read -t 1 line || exit 0; { echo "$$line" && cat; } | git apply -p2; } + +GOLANGCI_LINT_CONFIG := $(LINT_ROOT)/.golangci.yml +GOLANGCI_LINT_VERSION ?= v1.43.0 +GOLANGCI_LINT_BIN := out/linters/golangci-lint-$(GOLANGCI_LINT_VERSION)-$(LINT_ARCH) +$(GOLANGCI_LINT_BIN): + mkdir -p out/linters + rm -rf out/linters/golangci-lint-* + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b out/linters $(GOLANGCI_LINT_VERSION) + mv out/linters/golangci-lint $@ + +LINTERS += golangci-lint-lint +golangci-lint-lint: $(GOLANGCI_LINT_BIN) + find . -name go.mod -execdir "$(GOLANGCI_LINT_BIN)" run -c "$(GOLINT_CONFIG)" \; + +FIXERS += golangci-lint-fix +golangci-lint-fix: $(GOLANGCI_LINT_BIN) + find . -name go.mod -execdir "$(GOLANGCI_LINT_BIN)" run -c "$(GOLINT_CONFIG)" --fix \; + +YAMLLINT_VERSION ?= 1.26.3 +YAMLLINT_ROOT := out/linters/yamllint-$(YAMLLINT_VERSION) +YAMLLINT_BIN := $(YAMLLINT_ROOT)/dist/bin/yamllint +$(YAMLLINT_BIN): + mkdir -p out/linters + rm -rf out/linters/yamllint-* + curl -sSfL https://github.com/adrienverge/yamllint/archive/refs/tags/v$(YAMLLINT_VERSION).tar.gz | tar -C out/linters -zxf - + cd $(YAMLLINT_ROOT) && pip3 install --target dist . + +LINTERS += yamllint-lint +yamllint-lint: $(YAMLLINT_BIN) + PYTHONPATH=$(YAMLLINT_ROOT)/dist $(YAMLLINT_ROOT)/dist/bin/yamllint . + +.PHONY: _lint $(LINTERS) +_lint: $(LINTERS) + +.PHONY: fix $(FIXERS) +fix: $(FIXERS) + +# END: lint-install -makefile lint.mk . diff --git a/vendor/github.com/mistifyio/go-zfs/v3/rules.mk b/vendor/github.com/mistifyio/go-zfs/v3/rules.mk new file mode 100644 index 0000000000000..4746c978a6fd9 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/rules.mk @@ -0,0 +1,49 @@ +# Only use the recipes defined in these makefiles +MAKEFLAGS += --no-builtin-rules +.SUFFIXES: +# Delete target files if there's an error +# This avoids a failure to then skip building on next run if the output is created by shell redirection for example +# Not really necessary for now, but just good to have already if it becomes necessary later. +.DELETE_ON_ERROR: +# Treat the whole recipe as a one shell script/invocation instead of one-per-line +.ONESHELL: +# Use bash instead of plain sh +SHELL := bash +.SHELLFLAGS := -o pipefail -euc + +version := $(shell git rev-parse --short HEAD) +tag := $(shell git tag --points-at HEAD) +ifneq (,$(tag)) +version := $(tag)-$(version) +endif +LDFLAGS := -ldflags "-X main.version=$(version)" +export CGO_ENABLED := 0 + +ifeq ($(origin GOBIN), undefined) +GOBIN := ${PWD}/bin +export GOBIN +PATH := ${GOBIN}:${PATH} +export PATH +endif + +toolsBins := $(addprefix bin/,$(notdir $(shell grep '^\s*_' tooling/tools.go | awk -F'"' '{print $$2}'))) + +# installs cli tools defined in tools.go +$(toolsBins): tooling/go.mod tooling/go.sum tooling/tools.go +$(toolsBins): CMD=$(shell awk -F'"' '/$(@F)"/ {print $$2}' tooling/tools.go) +$(toolsBins): + cd tooling && go install $(CMD) + +.PHONY: gofumpt +gofumpt: bin/gofumpt + gofumpt -s -d . + +gofumpt-fix: bin/gofumpt + gofumpt -s -w . + +.PHONY: prettier prettier-fix +prettier: + prettier --list-different --ignore-path .gitignore . + +prettier-fix: + prettier --write --ignore-path .gitignore . diff --git a/vendor/github.com/mistifyio/go-zfs/v3/shell.nix b/vendor/github.com/mistifyio/go-zfs/v3/shell.nix new file mode 100644 index 0000000000000..e0ea24c16f31f --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/shell.nix @@ -0,0 +1,26 @@ +let _pkgs = import { }; +in { pkgs ? import (_pkgs.fetchFromGitHub { + owner = "NixOS"; + repo = "nixpkgs"; + #branch@date: 21.11@2022-02-13 + rev = "560ad8a2f89586ab1a14290f128ad6a393046065"; + sha256 = "0s0dv1clfpjyzy4p6ywxvzmwx9ddbr2yl77jf1wqdbr0x1206hb8"; +}) { } }: + +with pkgs; + +mkShell { + buildInputs = [ + git + gnumake + gnused + go + nixfmt + nodePackages.prettier + python3Packages.pip + python3Packages.setuptools + rufo + shfmt + vagrant + ]; +} diff --git a/vendor/github.com/mistifyio/go-zfs/utils.go b/vendor/github.com/mistifyio/go-zfs/v3/utils.go similarity index 72% rename from vendor/github.com/mistifyio/go-zfs/utils.go rename to vendor/github.com/mistifyio/go-zfs/v3/utils.go index c18c2c3dae6b1..b69942b530f48 100644 --- a/vendor/github.com/mistifyio/go-zfs/utils.go +++ b/vendor/github.com/mistifyio/go-zfs/v3/utils.go @@ -21,7 +21,6 @@ type command struct { } func (c *command) Run(arg ...string) ([][]string, error) { - cmd := exec.Command(c.Command, arg...) var stdout, stderr bytes.Buffer @@ -34,24 +33,24 @@ func (c *command) Run(arg ...string) ([][]string, error) { if c.Stdin != nil { cmd.Stdin = c.Stdin - } cmd.Stderr = &stderr id := uuid.New().String() - joinedArgs := strings.Join(cmd.Args, " ") + joinedArgs := cmd.Path + if len(cmd.Args) > 1 { + joinedArgs = strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " ") + } logger.Log([]string{"ID:" + id, "START", joinedArgs}) - err := cmd.Run() - logger.Log([]string{"ID:" + id, "FINISH"}) - - if err != nil { + if err := cmd.Run(); err != nil { return nil, &Error{ Err: err, - Debug: strings.Join([]string{cmd.Path, joinedArgs[1:]}, " "), + Debug: joinedArgs, Stderr: stderr.String(), } } + logger.Log([]string{"ID:" + id, "FINISH"}) // assume if you passed in something for stdout, that you know what to do with it if c.Stdout != nil { @@ -60,12 +59,12 @@ func (c *command) Run(arg ...string) ([][]string, error) { lines := strings.Split(stdout.String(), "\n") - //last line is always blank + // last line is always blank lines = lines[0 : len(lines)-1] output := make([][]string, len(lines)) for i, l := range lines { - output[i] = strings.Fields(l) + output[i] = strings.Split(l, "\t") } return output, nil @@ -92,33 +91,33 @@ func setUint(field *uint64, value string) error { return nil } -func (ds *Dataset) parseLine(line []string) error { +func (d *Dataset) parseLine(line []string) error { var err error if len(line) != len(dsPropList) { - return errors.New("Output does not match what is expected on this platform") + return errors.New("output does not match what is expected on this platform") } - setString(&ds.Name, line[0]) - setString(&ds.Origin, line[1]) + setString(&d.Name, line[0]) + setString(&d.Origin, line[1]) - if err = setUint(&ds.Used, line[2]); err != nil { + if err = setUint(&d.Used, line[2]); err != nil { return err } - if err = setUint(&ds.Avail, line[3]); err != nil { + if err = setUint(&d.Avail, line[3]); err != nil { return err } - setString(&ds.Mountpoint, line[4]) - setString(&ds.Compression, line[5]) - setString(&ds.Type, line[6]) + setString(&d.Mountpoint, line[4]) + setString(&d.Compression, line[5]) + setString(&d.Type, line[6]) - if err = setUint(&ds.Volsize, line[7]); err != nil { + if err = setUint(&d.Volsize, line[7]); err != nil { return err } - if err = setUint(&ds.Quota, line[8]); err != nil { + if err = setUint(&d.Quota, line[8]); err != nil { return err } - if err = setUint(&ds.Referenced, line[9]); err != nil { + if err = setUint(&d.Referenced, line[9]); err != nil { return err } @@ -126,17 +125,13 @@ func (ds *Dataset) parseLine(line []string) error { return nil } - if err = setUint(&ds.Written, line[10]); err != nil { + if err = setUint(&d.Written, line[10]); err != nil { return err } - if err = setUint(&ds.Logicalused, line[11]); err != nil { + if err = setUint(&d.Logicalused, line[11]); err != nil { return err } - if err = setUint(&ds.Usedbydataset, line[12]); err != nil { - return err - } - - return nil + return setUint(&d.Usedbydataset, line[12]) } /* @@ -156,12 +151,12 @@ func unescapeFilepath(path string) (string, error) { for i := 0; i < llen; { if path[i] == '\\' { if llen < i+4 { - return "", fmt.Errorf("Invalid octal code: too short") + return "", fmt.Errorf("invalid octal code: too short") } octalCode := path[(i + 1):(i + 4)] val, err := strconv.ParseUint(octalCode, 8, 8) if err != nil { - return "", fmt.Errorf("Invalid octal code: %v", err) + return "", fmt.Errorf("invalid octal code: %w", err) } buf = append(buf, byte(val)) i += 4 @@ -179,6 +174,7 @@ var changeTypeMap = map[string]ChangeType{ "M": Modified, "R": Renamed, } + var inodeTypeMap = map[string]InodeType{ "B": BlockDevice, "C": CharacterDevice, @@ -191,51 +187,51 @@ var inodeTypeMap = map[string]InodeType{ "F": File, } -// matches (+1) or (-1) -var referenceCountRegex = regexp.MustCompile("\\(([+-]\\d+?)\\)") +// matches (+1) or (-1). +var referenceCountRegex = regexp.MustCompile(`\(([+-]\d+?)\)`) func parseReferenceCount(field string) (int, error) { matches := referenceCountRegex.FindStringSubmatch(field) if matches == nil { - return 0, fmt.Errorf("Regexp does not match") + return 0, fmt.Errorf("regexp does not match") } return strconv.Atoi(matches[1]) } func parseInodeChange(line []string) (*InodeChange, error) { - llen := len(line) + llen := len(line) // nolint:ifshort // llen *is* actually used if llen < 1 { - return nil, fmt.Errorf("Empty line passed") + return nil, fmt.Errorf("empty line passed") } changeType := changeTypeMap[line[0]] if changeType == 0 { - return nil, fmt.Errorf("Unknown change type '%s'", line[0]) + return nil, fmt.Errorf("unknown change type '%s'", line[0]) } switch changeType { case Renamed: if llen != 4 { - return nil, fmt.Errorf("Mismatching number of fields: expect 4, got: %d", llen) + return nil, fmt.Errorf("mismatching number of fields: expect 4, got: %d", llen) } case Modified: if llen != 4 && llen != 3 { - return nil, fmt.Errorf("Mismatching number of fields: expect 3..4, got: %d", llen) + return nil, fmt.Errorf("mismatching number of fields: expect 3..4, got: %d", llen) } default: if llen != 3 { - return nil, fmt.Errorf("Mismatching number of fields: expect 3, got: %d", llen) + return nil, fmt.Errorf("mismatching number of fields: expect 3, got: %d", llen) } } inodeType := inodeTypeMap[line[1]] if inodeType == 0 { - return nil, fmt.Errorf("Unknown inode type '%s'", line[1]) + return nil, fmt.Errorf("unknown inode type '%s'", line[1]) } path, err := unescapeFilepath(line[2]) if err != nil { - return nil, fmt.Errorf("Failed to parse filename: %v", err) + return nil, fmt.Errorf("failed to parse filename: %w", err) } var newPath string @@ -244,13 +240,13 @@ func parseInodeChange(line []string) (*InodeChange, error) { case Renamed: newPath, err = unescapeFilepath(line[3]) if err != nil { - return nil, fmt.Errorf("Failed to parse filename: %v", err) + return nil, fmt.Errorf("failed to parse filename: %w", err) } case Modified: if llen == 4 { referenceCount, err = parseReferenceCount(line[3]) if err != nil { - return nil, fmt.Errorf("Failed to parse reference count: %v", err) + return nil, fmt.Errorf("failed to parse reference count: %w", err) } } default: @@ -266,18 +262,19 @@ func parseInodeChange(line []string) (*InodeChange, error) { }, nil } -// example input -//M / /testpool/bar/ -//+ F /testpool/bar/hello.txt -//M / /testpool/bar/hello.txt (+1) -//M / /testpool/bar/hello-hardlink +// example input for parseInodeChanges +// M / /testpool/bar/ +// + F /testpool/bar/hello.txt +// M / /testpool/bar/hello.txt (+1) +// M / /testpool/bar/hello-hardlink + func parseInodeChanges(lines [][]string) ([]*InodeChange, error) { changes := make([]*InodeChange, len(lines)) for i, line := range lines { c, err := parseInodeChange(line) if err != nil { - return nil, fmt.Errorf("Failed to parse line %d of zfs diff: %v, got: '%s'", i, err, line) + return nil, fmt.Errorf("failed to parse line %d of zfs diff: %w, got: '%s'", i, err, line) } changes[i] = c } @@ -290,7 +287,7 @@ func listByType(t, filter string) ([]*Dataset, error) { if filter != "" { args = append(args, filter) } - out, err := zfs(args...) + out, err := zfsOutput(args...) if err != nil { return nil, err } diff --git a/vendor/github.com/mistifyio/go-zfs/v3/utils_notsolaris.go b/vendor/github.com/mistifyio/go-zfs/v3/utils_notsolaris.go new file mode 100644 index 0000000000000..b1ce59656bc31 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/utils_notsolaris.go @@ -0,0 +1,19 @@ +//go:build !solaris +// +build !solaris + +package zfs + +import "strings" + +var ( + // List of ZFS properties to retrieve from zfs list command on a non-Solaris platform. + dsPropList = []string{"name", "origin", "used", "available", "mountpoint", "compression", "type", "volsize", "quota", "referenced", "written", "logicalused", "usedbydataset"} + + dsPropListOptions = strings.Join(dsPropList, ",") + + // List of Zpool properties to retrieve from zpool list command on a non-Solaris platform. + zpoolPropList = []string{"name", "health", "allocated", "size", "free", "readonly", "dedupratio", "fragmentation", "freeing", "leaked"} + + zpoolPropListOptions = strings.Join(zpoolPropList, ",") + zpoolArgs = []string{"get", "-Hp", zpoolPropListOptions} +) diff --git a/vendor/github.com/mistifyio/go-zfs/v3/utils_solaris.go b/vendor/github.com/mistifyio/go-zfs/v3/utils_solaris.go new file mode 100644 index 0000000000000..f19aebabb2aa6 --- /dev/null +++ b/vendor/github.com/mistifyio/go-zfs/v3/utils_solaris.go @@ -0,0 +1,19 @@ +//go:build solaris +// +build solaris + +package zfs + +import "strings" + +var ( + // List of ZFS properties to retrieve from zfs list command on a Solaris platform + dsPropList = []string{"name", "origin", "used", "available", "mountpoint", "compression", "type", "volsize", "quota", "referenced"} + + dsPropListOptions = strings.Join(dsPropList, ",") + + // List of Zpool properties to retrieve from zpool list command on a non-Solaris platform + zpoolPropList = []string{"name", "health", "allocated", "size", "free", "readonly", "dedupratio"} + + zpoolPropListOptions = strings.Join(zpoolPropList, ",") + zpoolArgs = []string{"get", "-Hp", zpoolPropListOptions} +) diff --git a/vendor/github.com/mistifyio/go-zfs/zfs.go b/vendor/github.com/mistifyio/go-zfs/v3/zfs.go similarity index 70% rename from vendor/github.com/mistifyio/go-zfs/zfs.go rename to vendor/github.com/mistifyio/go-zfs/v3/zfs.go index 4e5087ffe2885..1166bdc212ff5 100644 --- a/vendor/github.com/mistifyio/go-zfs/zfs.go +++ b/vendor/github.com/mistifyio/go-zfs/v3/zfs.go @@ -9,19 +9,18 @@ import ( "strings" ) -// ZFS dataset types, which can indicate if a dataset is a filesystem, -// snapshot, or volume. +// ZFS dataset types, which can indicate if a dataset is a filesystem, snapshot, or volume. const ( DatasetFilesystem = "filesystem" DatasetSnapshot = "snapshot" DatasetVolume = "volume" ) -// Dataset is a ZFS dataset. A dataset could be a clone, filesystem, snapshot, -// or volume. The Type struct member can be used to determine a dataset's type. +// Dataset is a ZFS dataset. A dataset could be a clone, filesystem, snapshot, or volume. +// The Type struct member can be used to determine a dataset's type. // // The field definitions can be found in the ZFS manual: -// http://www.freebsd.org/cgi/man.cgi?zfs(8). +// https://openzfs.github.io/openzfs-docs/man/7/zfsprops.7.html. type Dataset struct { Name string Origin string @@ -38,10 +37,10 @@ type Dataset struct { Referenced uint64 } -// InodeType is the type of inode as reported by Diff +// InodeType is the type of inode as reported by Diff. type InodeType int -// Types of Inodes +// Types of Inodes. const ( _ = iota // 0 == unknown type BlockDevice InodeType = iota @@ -55,10 +54,10 @@ const ( File ) -// ChangeType is the type of inode change as reported by Diff +// ChangeType is the type of inode change as reported by Diff. type ChangeType int -// Types of Changes +// Types of Changes. const ( _ = iota // 0 == unknown type Removed ChangeType = iota @@ -67,10 +66,10 @@ const ( Renamed ) -// DestroyFlag is the options flag passed to Destroy +// DestroyFlag is the options flag passed to Destroy. type DestroyFlag int -// Valid destroy options +// Valid destroy options. const ( DestroyDefault DestroyFlag = 1 << iota DestroyRecursive = 1 << iota @@ -79,7 +78,7 @@ const ( DestroyForceUmount = 1 << iota ) -// InodeChange represents a change as reported by Diff +// InodeChange represents a change as reported by Diff. type InodeChange struct { Change ChangeType Type InodeType @@ -88,65 +87,65 @@ type InodeChange struct { ReferenceCountChange int } -// Logger can be used to log commands/actions +// Logger can be used to log commands/actions. type Logger interface { Log(cmd []string) } type defaultLogger struct{} -func (*defaultLogger) Log(cmd []string) { - return +func (*defaultLogger) Log([]string) { } var logger Logger = &defaultLogger{} -// SetLogger set a log handler to log all commands including arguments before -// they are executed +// SetLogger set a log handler to log all commands including arguments before they are executed. func SetLogger(l Logger) { if l != nil { logger = l } } +// zfs is a helper function to wrap typical calls to zfs that ignores stdout. +func zfs(arg ...string) error { + _, err := zfsOutput(arg...) + return err +} + // zfs is a helper function to wrap typical calls to zfs. -func zfs(arg ...string) ([][]string, error) { +func zfsOutput(arg ...string) ([][]string, error) { c := command{Command: "zfs"} return c.Run(arg...) } // Datasets returns a slice of ZFS datasets, regardless of type. -// A filter argument may be passed to select a dataset with the matching name, -// or empty string ("") may be used to select all datasets. +// A filter argument may be passed to select a dataset with the matching name, or empty string ("") may be used to select all datasets. func Datasets(filter string) ([]*Dataset, error) { return listByType("all", filter) } // Snapshots returns a slice of ZFS snapshots. -// A filter argument may be passed to select a snapshot with the matching name, -// or empty string ("") may be used to select all snapshots. +// A filter argument may be passed to select a snapshot with the matching name, or empty string ("") may be used to select all snapshots. func Snapshots(filter string) ([]*Dataset, error) { return listByType(DatasetSnapshot, filter) } // Filesystems returns a slice of ZFS filesystems. -// A filter argument may be passed to select a filesystem with the matching name, -// or empty string ("") may be used to select all filesystems. +// A filter argument may be passed to select a filesystem with the matching name, or empty string ("") may be used to select all filesystems. func Filesystems(filter string) ([]*Dataset, error) { return listByType(DatasetFilesystem, filter) } // Volumes returns a slice of ZFS volumes. -// A filter argument may be passed to select a volume with the matching name, -// or empty string ("") may be used to select all volumes. +// A filter argument may be passed to select a volume with the matching name, or empty string ("") may be used to select all volumes. func Volumes(filter string) ([]*Dataset, error) { return listByType(DatasetVolume, filter) } -// GetDataset retrieves a single ZFS dataset by name. This dataset could be -// any valid ZFS dataset type, such as a clone, filesystem, snapshot, or volume. +// GetDataset retrieves a single ZFS dataset by name. +// This dataset could be any valid ZFS dataset type, such as a clone, filesystem, snapshot, or volume. func GetDataset(name string) (*Dataset, error) { - out, err := zfs("list", "-Hp", "-o", dsPropListOptions, name) + out, err := zfsOutput("list", "-Hp", "-o", dsPropListOptions, name) if err != nil { return nil, err } @@ -174,8 +173,7 @@ func (d *Dataset) Clone(dest string, properties map[string]string) (*Dataset, er args = append(args, propsSlice(properties)...) } args = append(args, []string{d.Name, dest}...) - _, err := zfs(args...) - if err != nil { + if err := zfs(args...); err != nil { return nil, err } return GetDataset(dest) @@ -192,8 +190,7 @@ func (d *Dataset) Unmount(force bool) (*Dataset, error) { args = append(args, "-f") } args = append(args, d.Name) - _, err := zfs(args...) - if err != nil { + if err := zfs(args...); err != nil { return nil, err } return GetDataset(d.Name) @@ -214,20 +211,17 @@ func (d *Dataset) Mount(overlay bool, options []string) (*Dataset, error) { args = append(args, strings.Join(options, ",")) } args = append(args, d.Name) - _, err := zfs(args...) - if err != nil { + if err := zfs(args...); err != nil { return nil, err } return GetDataset(d.Name) } -// ReceiveSnapshot receives a ZFS stream from the input io.Reader, creates a -// new snapshot with the specified name, and streams the input data into the -// newly-created snapshot. +// ReceiveSnapshot receives a ZFS stream from the input io.Reader. +// A new snapshot is created with the specified name, and streams the input data into the newly-created snapshot. func ReceiveSnapshot(input io.Reader, name string) (*Dataset, error) { c := command{Command: "zfs", Stdin: input} - _, err := c.Run("receive", name) - if err != nil { + if _, err := c.Run("receive", name); err != nil { return nil, err } return GetDataset(name) @@ -245,10 +239,21 @@ func (d *Dataset) SendSnapshot(output io.Writer) error { return err } -// CreateVolume creates a new ZFS volume with the specified name, size, and -// properties. -// A full list of available ZFS properties may be found here: -// https://www.freebsd.org/cgi/man.cgi?zfs(8). +// IncrementalSend sends a ZFS stream of a snapshot to the input io.Writer using the baseSnapshot as the starting point. +// An error will be returned if the input dataset is not of snapshot type. +func (d *Dataset) IncrementalSend(baseSnapshot *Dataset, output io.Writer) error { + if d.Type != DatasetSnapshot || baseSnapshot.Type != DatasetSnapshot { + return errors.New("can only send snapshots") + } + c := command{Command: "zfs", Stdout: output} + _, err := c.Run("send", "-i", baseSnapshot.Name, d.Name) + return err +} + +// CreateVolume creates a new ZFS volume with the specified name, size, and properties. +// +// A full list of available ZFS properties may be found in the ZFS manual: +// https://openzfs.github.io/openzfs-docs/man/7/zfsprops.7.html. func CreateVolume(name string, size uint64, properties map[string]string) (*Dataset, error) { args := make([]string, 4, 5) args[0] = "create" @@ -259,17 +264,15 @@ func CreateVolume(name string, size uint64, properties map[string]string) (*Data args = append(args, propsSlice(properties)...) } args = append(args, name) - _, err := zfs(args...) - if err != nil { + if err := zfs(args...); err != nil { return nil, err } return GetDataset(name) } -// Destroy destroys a ZFS dataset. If the destroy bit flag is set, any -// descendents of the dataset will be recursively destroyed, including snapshots. -// If the deferred bit flag is set, the snapshot is marked for deferred -// deletion. +// Destroy destroys a ZFS dataset. +// If the destroy bit flag is set, any descendents of the dataset will be recursively destroyed, including snapshots. +// If the deferred bit flag is set, the snapshot is marked for deferred deletion. func (d *Dataset) Destroy(flags DestroyFlag) error { args := make([]string, 1, 3) args[0] = "destroy" @@ -290,25 +293,26 @@ func (d *Dataset) Destroy(flags DestroyFlag) error { } args = append(args, d.Name) - _, err := zfs(args...) + err := zfs(args...) return err } // SetProperty sets a ZFS property on the receiving dataset. -// A full list of available ZFS properties may be found here: -// https://www.freebsd.org/cgi/man.cgi?zfs(8). +// +// A full list of available ZFS properties may be found in the ZFS manual: +// https://openzfs.github.io/openzfs-docs/man/7/zfsprops.7.html. func (d *Dataset) SetProperty(key, val string) error { prop := strings.Join([]string{key, val}, "=") - _, err := zfs("set", prop, d.Name) + err := zfs("set", prop, d.Name) return err } -// GetProperty returns the current value of a ZFS property from the -// receiving dataset. -// A full list of available ZFS properties may be found here: -// https://www.freebsd.org/cgi/man.cgi?zfs(8). +// GetProperty returns the current value of a ZFS property from the receiving dataset. +// +// A full list of available ZFS properties may be found in the ZFS manual: +// https://openzfs.github.io/openzfs-docs/man/7/zfsprops.7.html. func (d *Dataset) GetProperty(key string) (string, error) { - out, err := zfs("get", "-H", key, d.Name) + out, err := zfsOutput("get", "-H", key, d.Name) if err != nil { return "", err } @@ -317,7 +321,7 @@ func (d *Dataset) GetProperty(key string) (string, error) { } // Rename renames a dataset. -func (d *Dataset) Rename(name string, createParent bool, recursiveRenameSnapshots bool) (*Dataset, error) { +func (d *Dataset) Rename(name string, createParent, recursiveRenameSnapshots bool) (*Dataset, error) { args := make([]string, 3, 5) args[0] = "rename" args[1] = d.Name @@ -328,8 +332,7 @@ func (d *Dataset) Rename(name string, createParent bool, recursiveRenameSnapshot if recursiveRenameSnapshots { args = append(args, "-r") } - _, err := zfs(args...) - if err != nil { + if err := zfs(args...); err != nil { return d, err } @@ -341,10 +344,10 @@ func (d *Dataset) Snapshots() ([]*Dataset, error) { return Snapshots(d.Name) } -// CreateFilesystem creates a new ZFS filesystem with the specified name and -// properties. -// A full list of available ZFS properties may be found here: -// https://www.freebsd.org/cgi/man.cgi?zfs(8). +// CreateFilesystem creates a new ZFS filesystem with the specified name and properties. +// +// A full list of available ZFS properties may be found in the ZFS manual: +// https://openzfs.github.io/openzfs-docs/man/7/zfsprops.7.html. func CreateFilesystem(name string, properties map[string]string) (*Dataset, error) { args := make([]string, 1, 4) args[0] = "create" @@ -354,16 +357,14 @@ func CreateFilesystem(name string, properties map[string]string) (*Dataset, erro } args = append(args, name) - _, err := zfs(args...) - if err != nil { + if err := zfs(args...); err != nil { return nil, err } return GetDataset(name) } -// Snapshot creates a new ZFS snapshot of the receiving dataset, using the -// specified name. Optionally, the snapshot can be taken recursively, creating -// snapshots of all descendent filesystems in a single, atomic operation. +// Snapshot creates a new ZFS snapshot of the receiving dataset, using the specified name. +// Optionally, the snapshot can be taken recursively, creating snapshots of all descendent filesystems in a single, atomic operation. func (d *Dataset) Snapshot(name string, recursive bool) (*Dataset, error) { args := make([]string, 1, 4) args[0] = "snapshot" @@ -372,17 +373,15 @@ func (d *Dataset) Snapshot(name string, recursive bool) (*Dataset, error) { } snapName := fmt.Sprintf("%s@%s", d.Name, name) args = append(args, snapName) - _, err := zfs(args...) - if err != nil { + if err := zfs(args...); err != nil { return nil, err } return GetDataset(snapName) } // Rollback rolls back the receiving ZFS dataset to a previous snapshot. -// Optionally, intermediate snapshots can be destroyed. A ZFS snapshot -// rollback cannot be completed without this option, if more recent -// snapshots exist. +// Optionally, intermediate snapshots can be destroyed. +// A ZFS snapshot rollback cannot be completed without this option, if more recent snapshots exist. // An error will be returned if the input dataset is not of snapshot type. func (d *Dataset) Rollback(destroyMoreRecent bool) error { if d.Type != DatasetSnapshot { @@ -396,13 +395,12 @@ func (d *Dataset) Rollback(destroyMoreRecent bool) error { } args = append(args, d.Name) - _, err := zfs(args...) + err := zfs(args...) return err } // Children returns a slice of children of the receiving ZFS dataset. -// A recursion depth may be specified, or a depth of 0 allows unlimited -// recursion. +// A recursion depth may be specified, or a depth of 0 allows unlimited recursion. func (d *Dataset) Children(depth uint64) ([]*Dataset, error) { args := []string{"list"} if depth > 0 { @@ -414,7 +412,7 @@ func (d *Dataset) Children(depth uint64) ([]*Dataset, error) { args = append(args, "-t", "all", "-Hp", "-o", dsPropListOptions) args = append(args, d.Name) - out, err := zfs(args...) + out, err := zfsOutput(args...) if err != nil { return nil, err } @@ -436,11 +434,10 @@ func (d *Dataset) Children(depth uint64) ([]*Dataset, error) { } // Diff returns changes between a snapshot and the given ZFS dataset. -// The snapshot name must include the filesystem part as it is possible to -// compare clones with their origin snapshots. +// The snapshot name must include the filesystem part as it is possible to compare clones with their origin snapshots. func (d *Dataset) Diff(snapshot string) ([]*InodeChange, error) { - args := []string{"diff", "-FH", snapshot, d.Name}[:] - out, err := zfs(args...) + args := []string{"diff", "-FH", snapshot, d.Name} + out, err := zfsOutput(args...) if err != nil { return nil, err } diff --git a/vendor/github.com/mistifyio/go-zfs/zpool.go b/vendor/github.com/mistifyio/go-zfs/v3/zpool.go similarity index 70% rename from vendor/github.com/mistifyio/go-zfs/zpool.go rename to vendor/github.com/mistifyio/go-zfs/v3/zpool.go index d8db945d708c6..a0bd6471a5bba 100644 --- a/vendor/github.com/mistifyio/go-zfs/zpool.go +++ b/vendor/github.com/mistifyio/go-zfs/v3/zpool.go @@ -1,8 +1,9 @@ package zfs -// ZFS zpool states, which can indicate if a pool is online, offline, -// degraded, etc. More information regarding zpool states can be found here: -// https://docs.oracle.com/cd/E19253-01/819-5461/gamno/index.html. +// ZFS zpool states, which can indicate if a pool is online, offline, degraded, etc. +// +// More information regarding zpool states can be found in the ZFS manual: +// https://openzfs.github.io/openzfs-docs/man/7/zpoolconcepts.7.html#Device_Failure_and_Recovery const ( ZpoolOnline = "ONLINE" ZpoolDegraded = "DEGRADED" @@ -12,8 +13,8 @@ const ( ZpoolRemoved = "REMOVED" ) -// Zpool is a ZFS zpool. A pool is a top-level structure in ZFS, and can -// contain many descendent datasets. +// Zpool is a ZFS zpool. +// A pool is a top-level structure in ZFS, and can contain many descendent datasets. type Zpool struct { Name string Health string @@ -27,8 +28,14 @@ type Zpool struct { DedupRatio float64 } +// zpool is a helper function to wrap typical calls to zpool and ignores stdout. +func zpool(arg ...string) error { + _, err := zpoolOutput(arg...) + return err +} + // zpool is a helper function to wrap typical calls to zpool. -func zpool(arg ...string) ([][]string, error) { +func zpoolOutput(arg ...string) ([][]string, error) { c := command{Command: "zpool"} return c.Run(arg...) } @@ -37,14 +44,11 @@ func zpool(arg ...string) ([][]string, error) { func GetZpool(name string) (*Zpool, error) { args := zpoolArgs args = append(args, name) - out, err := zpool(args...) + out, err := zpoolOutput(args...) if err != nil { return nil, err } - // there is no -H - out = out[1:] - z := &Zpool{Name: name} for _, line := range out { if err := z.parseLine(line); err != nil { @@ -65,10 +69,11 @@ func (z *Zpool) Snapshots() ([]*Dataset, error) { return Snapshots(z.Name) } -// CreateZpool creates a new ZFS zpool with the specified name, properties, -// and optional arguments. -// A full list of available ZFS properties and command-line arguments may be -// found here: https://www.freebsd.org/cgi/man.cgi?zfs(8). +// CreateZpool creates a new ZFS zpool with the specified name, properties, and optional arguments. +// +// A full list of available ZFS properties and command-line arguments may be found in the ZFS manual: +// https://openzfs.github.io/openzfs-docs/man/7/zfsprops.7.html. +// https://openzfs.github.io/openzfs-docs/man/8/zpool-create.8.html func CreateZpool(name string, properties map[string]string, args ...string) (*Zpool, error) { cli := make([]string, 1, 4) cli[0] = "create" @@ -77,8 +82,7 @@ func CreateZpool(name string, properties map[string]string, args ...string) (*Zp } cli = append(cli, name) cli = append(cli, args...) - _, err := zpool(cli...) - if err != nil { + if err := zpool(cli...); err != nil { return nil, err } @@ -87,14 +91,14 @@ func CreateZpool(name string, properties map[string]string, args ...string) (*Zp // Destroy destroys a ZFS zpool by name. func (z *Zpool) Destroy() error { - _, err := zpool("destroy", z.Name) + err := zpool("destroy", z.Name) return err } // ListZpools list all ZFS zpools accessible on the current system. func ListZpools() ([]*Zpool, error) { args := []string{"list", "-Ho", "name"} - out, err := zpool(args...) + out, err := zpoolOutput(args...) if err != nil { return nil, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 380f42fa0d493..44a2cdff77381 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -570,9 +570,9 @@ github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/miekg/dns v1.1.43 ## explicit; go 1.14 github.com/miekg/dns -# github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible -## explicit -github.com/mistifyio/go-zfs +# github.com/mistifyio/go-zfs/v3 v3.0.1 +## explicit; go 1.14 +github.com/mistifyio/go-zfs/v3 # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 From e370f224ae7f94fea0554e32125926a3caeec344 Mon Sep 17 00:00:00 2001 From: Jeyanthinath Muthuram Date: Mon, 8 May 2023 15:27:52 +0530 Subject: [PATCH 052/293] fixing consistent aliases for OCI spec imports Signed-off-by: Jeyanthinath Muthuram (cherry picked from commit 307b09e7eb14e48a66752b036fb2a2eb53fa599c) Signed-off-by: Sebastiaan van Stijn --- .../router/container/container_routes.go | 4 +- .../distribution/distribution_routes.go | 10 ++--- api/server/router/image/backend.go | 6 +-- api/server/router/image/image_routes.go | 4 +- api/types/backend/build.go | 4 +- api/types/configs.go | 4 +- api/types/image/opts.go | 4 +- api/types/registry/registry.go | 6 +-- .../adapters/localinlinecache/inlinecache.go | 12 +++--- builder/dockerfile/builder.go | 4 +- builder/dockerfile/copy.go | 6 +-- builder/dockerfile/dispatchers.go | 8 ++-- builder/dockerfile/imagecontext.go | 10 ++--- builder/dockerfile/internals.go | 4 +- client/container_create.go | 6 +-- client/interface.go | 4 +- client/service_create_test.go | 8 ++-- daemon/cluster/executor/backend.go | 4 +- daemon/containerd/image_list.go | 14 +++---- daemon/containerd/image_pull.go | 6 +-- daemon/containerd/image_snapshot.go | 4 +- daemon/create.go | 8 ++-- daemon/image_service.go | 10 ++--- daemon/images/image.go | 29 +++++++------- daemon/images/image_builder.go | 6 +-- daemon/images/image_import.go | 4 +- daemon/images/image_pull.go | 6 +-- daemon/images/images_test.go | 10 ++--- daemon/images/store_test.go | 4 +- distribution/config.go | 8 ++-- distribution/manifest.go | 18 ++++----- distribution/manifest_test.go | 24 +++++------ distribution/pull_v2.go | 40 +++++++++---------- distribution/pull_v2_test.go | 4 +- distribution/pull_v2_unix.go | 8 ++-- distribution/pull_v2_windows.go | 6 +-- integration/container/create_test.go | 6 +-- integration/image/pull_test.go | 18 ++++----- integration/internal/container/container.go | 4 +- integration/internal/container/ops.go | 4 +- integration/plugin/common/plugin_test.go | 4 +- libcontainerd/remote/client.go | 6 +-- plugin/backend_linux.go | 28 ++++++------- plugin/fetch_linux.go | 30 +++++++------- plugin/manager_linux.go | 4 +- 45 files changed, 210 insertions(+), 211 deletions(-) diff --git a/api/server/router/container/container_routes.go b/api/server/router/container/container_routes.go index 5978880124939..b4aa0864fb4e3 100644 --- a/api/server/router/container/container_routes.go +++ b/api/server/router/container/container_routes.go @@ -21,7 +21,7 @@ import ( containerpkg "github.com/docker/docker/container" "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/ioutils" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/websocket" @@ -569,7 +569,7 @@ func (s *containerRouter) postContainersCreate(ctx context.Context, w http.Respo hostConfig.Annotations = nil } - var platform *specs.Platform + var platform *ocispec.Platform if versions.GreaterThanOrEqualTo(version, "1.41") { if v := r.Form.Get("platform"); v != "" { p, err := platforms.Parse(v) diff --git a/api/server/router/distribution/distribution_routes.go b/api/server/router/distribution/distribution_routes.go index 89d120f39e117..319affb9dd000 100644 --- a/api/server/router/distribution/distribution_routes.go +++ b/api/server/router/distribution/distribution_routes.go @@ -12,7 +12,7 @@ import ( "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types/registry" "github.com/docker/docker/errdefs" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -61,7 +61,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res if err != nil { return err } - distributionInspect.Descriptor = v1.Descriptor{ + distributionInspect.Descriptor = ocispec.Descriptor{ MediaType: descriptor.MediaType, Digest: descriptor.Digest, Size: descriptor.Size, @@ -107,7 +107,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res switch mnfstObj := mnfst.(type) { case *manifestlist.DeserializedManifestList: for _, m := range mnfstObj.Manifests { - distributionInspect.Platforms = append(distributionInspect.Platforms, v1.Platform{ + distributionInspect.Platforms = append(distributionInspect.Platforms, ocispec.Platform{ Architecture: m.Platform.Architecture, OS: m.Platform.OS, OSVersion: m.Platform.OSVersion, @@ -117,7 +117,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res } case *schema2.DeserializedManifest: configJSON, err := blobsrvc.Get(ctx, mnfstObj.Config.Digest) - var platform v1.Platform + var platform ocispec.Platform if err == nil { err := json.Unmarshal(configJSON, &platform) if err == nil && (platform.OS != "" || platform.Architecture != "") { @@ -125,7 +125,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res } } case *schema1.SignedManifest: - platform := v1.Platform{ + platform := ocispec.Platform{ Architecture: mnfstObj.Architecture, OS: "linux", } diff --git a/api/server/router/image/backend.go b/api/server/router/image/backend.go index f7cacba90f144..c8c01d2102fd2 100644 --- a/api/server/router/image/backend.go +++ b/api/server/router/image/backend.go @@ -10,7 +10,7 @@ import ( "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/registry" dockerimage "github.com/docker/docker/image" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // Backend is all the methods that need to be implemented @@ -32,12 +32,12 @@ type imageBackend interface { type importExportBackend interface { LoadImage(ctx context.Context, inTar io.ReadCloser, outStream io.Writer, quiet bool) error - ImportImage(ctx context.Context, ref reference.Named, platform *specs.Platform, msg string, layerReader io.Reader, changes []string) (dockerimage.ID, error) + ImportImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (dockerimage.ID, error) ExportImage(ctx context.Context, names []string, outStream io.Writer) error } type registryBackend interface { - PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error + PullImage(ctx context.Context, image, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error PushImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error } diff --git a/api/server/router/image/image_routes.go b/api/server/router/image/image_routes.go index a7f8cc9f91ef8..483bcbe9f3c2c 100644 --- a/api/server/router/image/image_routes.go +++ b/api/server/router/image/image_routes.go @@ -24,7 +24,7 @@ import ( "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/streamformatter" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -41,7 +41,7 @@ func (ir *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrit comment = r.Form.Get("message") progressErr error output = ioutils.NewWriteFlusher(w) - platform *specs.Platform + platform *ocispec.Platform ) defer output.Close() diff --git a/api/types/backend/build.go b/api/types/backend/build.go index 9f1348e12cc72..91715d0b91b4f 100644 --- a/api/types/backend/build.go +++ b/api/types/backend/build.go @@ -6,7 +6,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/registry" "github.com/docker/docker/pkg/streamformatter" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // PullOption defines different modes for accessing images @@ -42,5 +42,5 @@ type GetImageAndLayerOptions struct { PullOption PullOption AuthConfig map[string]registry.AuthConfig Output io.Writer - Platform *specs.Platform + Platform *ocispec.Platform } diff --git a/api/types/configs.go b/api/types/configs.go index 7689f38b331f2..7d5930bbeb65c 100644 --- a/api/types/configs.go +++ b/api/types/configs.go @@ -3,7 +3,7 @@ package types // import "github.com/docker/docker/api/types" import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // configs holds structs used for internal communication between the @@ -16,7 +16,7 @@ type ContainerCreateConfig struct { Config *container.Config HostConfig *container.HostConfig NetworkingConfig *network.NetworkingConfig - Platform *specs.Platform + Platform *ocispec.Platform AdjustCPUShares bool } diff --git a/api/types/image/opts.go b/api/types/image/opts.go index a24f9059ab4f7..3cefecb0da347 100644 --- a/api/types/image/opts.go +++ b/api/types/image/opts.go @@ -1,9 +1,9 @@ package image -import specs "github.com/opencontainers/image-spec/specs-go/v1" +import ocispec "github.com/opencontainers/image-spec/specs-go/v1" // GetImageOpts holds parameters to inspect an image. type GetImageOpts struct { - Platform *specs.Platform + Platform *ocispec.Platform Details bool } diff --git a/api/types/registry/registry.go b/api/types/registry/registry.go index 62a88f5be89d5..b83f5d7b2e29b 100644 --- a/api/types/registry/registry.go +++ b/api/types/registry/registry.go @@ -4,7 +4,7 @@ import ( "encoding/json" "net" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // ServiceConfig stores daemon registry services configuration. @@ -113,8 +113,8 @@ type SearchResults struct { type DistributionInspect struct { // Descriptor contains information about the manifest, including // the content addressable digest - Descriptor v1.Descriptor + Descriptor ocispec.Descriptor // Platforms contains the list of platforms supported by the image, // obtained by parsing the manifest - Platforms []v1.Platform + Platforms []ocispec.Platform } diff --git a/builder/builder-next/adapters/localinlinecache/inlinecache.go b/builder/builder-next/adapters/localinlinecache/inlinecache.go index 6c00852c5f0eb..0d1e94df31445 100644 --- a/builder/builder-next/adapters/localinlinecache/inlinecache.go +++ b/builder/builder-next/adapters/localinlinecache/inlinecache.go @@ -18,7 +18,7 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/worker" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -26,9 +26,9 @@ import ( func ResolveCacheImporterFunc(sm *session.Manager, resolverFunc docker.RegistryHosts, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc { upstream := registryremotecache.ResolveCacheImporterFunc(sm, cs, resolverFunc) - return func(ctx context.Context, group session.Group, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) { + return func(ctx context.Context, group session.Group, attrs map[string]string) (remotecache.Importer, ocispec.Descriptor, error) { if dt, err := tryImportLocal(rs, is, attrs["ref"]); err == nil { - return newLocalImporter(dt), specs.Descriptor{}, nil + return newLocalImporter(dt), ocispec.Descriptor{}, nil } return upstream(ctx, group, attrs) } @@ -59,7 +59,7 @@ type localImporter struct { dt []byte } -func (li *localImporter) Resolve(ctx context.Context, _ specs.Descriptor, id string, w worker.Worker) (solver.CacheManager, error) { +func (li *localImporter) Resolve(ctx context.Context, _ ocispec.Descriptor, id string, w worker.Worker) (solver.CacheManager, error) { cc := v1.NewCacheChains() if err := li.importInlineCache(ctx, li.dt, cc); err != nil { return nil, err @@ -96,7 +96,7 @@ func (li *localImporter) importInlineCache(ctx context.Context, dt []byte, cc so layers := v1.DescriptorProvider{} for i, diffID := range img.Rootfs.DiffIDs { dgst := digest.Digest(diffID.String()) - desc := specs.Descriptor{ + desc := ocispec.Descriptor{ Digest: dgst, Size: -1, MediaType: images.MediaTypeDockerSchema2Layer, @@ -157,6 +157,6 @@ func parseCreatedLayerInfo(img image) ([]string, []string, error) { type emptyProvider struct { } -func (p *emptyProvider) ReaderAt(ctx context.Context, dec specs.Descriptor) (content.ReaderAt, error) { +func (p *emptyProvider) ReaderAt(ctx context.Context, dec ocispec.Descriptor) (content.ReaderAt, error) { return nil, errors.Errorf("ReaderAt not implemented for empty provider") } diff --git a/builder/dockerfile/builder.go b/builder/dockerfile/builder.go index e515c698c0c4a..04cc07337e4c0 100644 --- a/builder/dockerfile/builder.go +++ b/builder/dockerfile/builder.go @@ -21,7 +21,7 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sync/syncmap" @@ -125,7 +125,7 @@ type Builder struct { pathCache pathCache containerManager *containerManager imageProber ImageProber - platform *specs.Platform + platform *ocispec.Platform } // newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options. diff --git a/builder/dockerfile/copy.go b/builder/dockerfile/copy.go index 0a994067fcb22..7919c972fdf29 100644 --- a/builder/dockerfile/copy.go +++ b/builder/dockerfile/copy.go @@ -24,7 +24,7 @@ import ( "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/system" "github.com/moby/buildkit/frontend/dockerfile/instructions" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -74,7 +74,7 @@ type copier struct { source builder.Source pathCache pathCache download sourceDownloader - platform *specs.Platform + platform *ocispec.Platform // for cleanup. TODO: having copier.cleanup() is error prone and hard to // follow. Code calling performCopy should manage the lifecycle of its params. // Copier should take override source as input, not imageMount. @@ -86,7 +86,7 @@ func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, i platform := req.builder.platform if platform == nil { // May be nil if not explicitly set in API/dockerfile - platform = &specs.Platform{} + platform = &ocispec.Platform{} } if platform.OS == "" { // Default to the dispatch requests operating system if not explicit in API/dockerfile diff --git a/builder/dockerfile/dispatchers.go b/builder/dockerfile/dispatchers.go index 9675134fa6479..663456734ee40 100644 --- a/builder/dockerfile/dispatchers.go +++ b/builder/dockerfile/dispatchers.go @@ -28,7 +28,7 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" "github.com/moby/sys/signal" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -158,7 +158,7 @@ func initializeStage(ctx context.Context, d dispatchRequest, cmd *instructions.S return err } - var platform *specs.Platform + var platform *ocispec.Platform if v := cmd.Platform; v != "" { v, err := d.getExpandedString(d.shlex, v) if err != nil { @@ -232,7 +232,7 @@ func (d *dispatchRequest) getExpandedString(shlex *shell.Lex, str string) (strin return name, nil } -func (d *dispatchRequest) getImageOrStage(ctx context.Context, name string, platform *specs.Platform) (builder.Image, error) { +func (d *dispatchRequest) getImageOrStage(ctx context.Context, name string, platform *ocispec.Platform) (builder.Image, error) { var localOnly bool if im, ok := d.stages.getByName(name); ok { name = im.Image @@ -266,7 +266,7 @@ func (d *dispatchRequest) getImageOrStage(ctx context.Context, name string, plat return imageMount.Image(), nil } -func (d *dispatchRequest) getFromImage(ctx context.Context, shlex *shell.Lex, basename string, platform *specs.Platform) (builder.Image, error) { +func (d *dispatchRequest) getFromImage(ctx context.Context, shlex *shell.Lex, basename string, platform *ocispec.Platform) (builder.Image, error) { name, err := d.getExpandedString(shlex, basename) if err != nil { return nil, err diff --git a/builder/dockerfile/imagecontext.go b/builder/dockerfile/imagecontext.go index ced18d44218e9..7dada665965aa 100644 --- a/builder/dockerfile/imagecontext.go +++ b/builder/dockerfile/imagecontext.go @@ -8,12 +8,12 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" dockerimage "github.com/docker/docker/image" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -type getAndMountFunc func(context.Context, string, bool, *specs.Platform) (builder.Image, builder.ROLayer, error) +type getAndMountFunc func(context.Context, string, bool, *ocispec.Platform) (builder.Image, builder.ROLayer, error) // imageSources mounts images and provides a cache for mounted images. It tracks // all images so they can be unmounted at the end of the build. @@ -24,7 +24,7 @@ type imageSources struct { } func newImageSources(options builderOptions) *imageSources { - getAndMount := func(ctx context.Context, idOrRef string, localOnly bool, platform *specs.Platform) (builder.Image, builder.ROLayer, error) { + getAndMount := func(ctx context.Context, idOrRef string, localOnly bool, platform *ocispec.Platform) (builder.Image, builder.ROLayer, error) { pullOption := backend.PullOptionNoPull if !localOnly { if options.Options.PullParent { @@ -47,7 +47,7 @@ func newImageSources(options builderOptions) *imageSources { } } -func (m *imageSources) Get(ctx context.Context, idOrRef string, localOnly bool, platform *specs.Platform) (*imageMount, error) { +func (m *imageSources) Get(ctx context.Context, idOrRef string, localOnly bool, platform *ocispec.Platform) (*imageMount, error) { if im, ok := m.byImageID[idOrRef]; ok { return im, nil } @@ -71,7 +71,7 @@ func (m *imageSources) Unmount() (retErr error) { return } -func (m *imageSources) Add(im *imageMount, platform *specs.Platform) { +func (m *imageSources) Add(im *imageMount, platform *ocispec.Platform) { switch im.image { case nil: // Set the platform for scratch images diff --git a/builder/dockerfile/internals.go b/builder/dockerfile/internals.go index 6aa96a972cd92..050deb1aad34b 100644 --- a/builder/dockerfile/internals.go +++ b/builder/dockerfile/internals.go @@ -19,7 +19,7 @@ import ( "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/stringid" "github.com/docker/go-connections/nat" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -74,7 +74,7 @@ func (b *Builder) exportImage(ctx context.Context, state *dispatchState, layer b return errors.Errorf("unexpected image type") } - platform := &specs.Platform{ + platform := &ocispec.Platform{ OS: parentImage.OS, Architecture: parentImage.Architecture, Variant: parentImage.Variant, diff --git a/client/container_create.go b/client/container_create.go index f82420b673ec2..193a2bb56264c 100644 --- a/client/container_create.go +++ b/client/container_create.go @@ -9,7 +9,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/versions" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) type configWrapper struct { @@ -20,7 +20,7 @@ type configWrapper struct { // ContainerCreate creates a new container based on the given configuration. // It can be associated with a name, but it's not mandatory. -func (cli *Client) ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.CreateResponse, error) { +func (cli *Client) ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *ocispec.Platform, containerName string) (container.CreateResponse, error) { var response container.CreateResponse if err := cli.NewVersionError("1.25", "stop timeout"); config != nil && config.StopTimeout != nil && err != nil { @@ -75,7 +75,7 @@ func (cli *Client) ContainerCreate(ctx context.Context, config *container.Config // Similar to containerd's platforms.Format(), but does allow components to be // omitted (e.g. pass "architecture" only, without "os": // https://github.com/containerd/containerd/blob/v1.5.2/platforms/platforms.go#L243-L263 -func formatPlatform(platform *specs.Platform) string { +func formatPlatform(platform *ocispec.Platform) string { if platform == nil { return "" } diff --git a/client/interface.go b/client/interface.go index 64877d1641651..7993c5a48fa82 100644 --- a/client/interface.go +++ b/client/interface.go @@ -15,7 +15,7 @@ import ( "github.com/docker/docker/api/types/registry" "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/api/types/volume" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // CommonAPIClient is the common methods between stable and experimental versions of APIClient. @@ -47,7 +47,7 @@ type CommonAPIClient interface { type ContainerAPIClient interface { ContainerAttach(ctx context.Context, container string, options types.ContainerAttachOptions) (types.HijackedResponse, error) ContainerCommit(ctx context.Context, container string, options types.ContainerCommitOptions) (types.IDResponse, error) - ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.CreateResponse, error) + ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *ocispec.Platform, containerName string) (container.CreateResponse, error) ContainerDiff(ctx context.Context, container string) ([]container.FilesystemChange, error) ContainerExecAttach(ctx context.Context, execID string, config types.ExecStartCheck) (types.HijackedResponse, error) ContainerExecCreate(ctx context.Context, container string, config types.ExecConfig) (types.IDResponse, error) diff --git a/client/service_create_test.go b/client/service_create_test.go index e7294f85047f9..f75891bd4002a 100644 --- a/client/service_create_test.go +++ b/client/service_create_test.go @@ -15,7 +15,7 @@ import ( "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/errdefs" "github.com/opencontainers/go-digest" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" ) @@ -91,10 +91,10 @@ func TestServiceCreateCompatiblePlatforms(t *testing.T) { }, nil } else if strings.HasPrefix(req.URL.Path, "/v1.30/distribution/") { b, err := json.Marshal(registrytypes.DistributionInspect{ - Descriptor: v1.Descriptor{ + Descriptor: ocispec.Descriptor{ Digest: "sha256:c0537ff6a5218ef531ece93d4984efc99bbf3f7497c0a7726c88e2bb7584dc96", }, - Platforms: []v1.Platform{ + Platforms: []ocispec.Platform{ { Architecture: "amd64", OS: "linux", @@ -171,7 +171,7 @@ func TestServiceCreateDigestPinning(t *testing.T) { } else if strings.HasPrefix(req.URL.Path, "/v1.30/distribution/") { // resolvable images b, err := json.Marshal(registrytypes.DistributionInspect{ - Descriptor: v1.Descriptor{ + Descriptor: ocispec.Descriptor{ Digest: digest.Digest(dgst), }, }) diff --git a/daemon/cluster/executor/backend.go b/daemon/cluster/executor/backend.go index dd3f513364b7a..91c243ab8ecff 100644 --- a/daemon/cluster/executor/backend.go +++ b/daemon/cluster/executor/backend.go @@ -27,7 +27,7 @@ import ( "github.com/docker/docker/plugin" volumeopts "github.com/docker/docker/volume/service/opts" "github.com/moby/swarmkit/v2/agent/exec" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // Backend defines the executor component for a swarm agent. @@ -75,7 +75,7 @@ type VolumeBackend interface { // ImageBackend is used by an executor to perform image operations type ImageBackend interface { - PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error + PullImage(ctx context.Context, image, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error GetRepository(context.Context, reference.Named, *registry.AuthConfig) (distribution.Repository, error) GetImage(ctx context.Context, refOrID string, options opts.GetImageOpts) (*image.Image, error) } diff --git a/daemon/containerd/image_list.go b/daemon/containerd/image_list.go index 9128f88797d7c..a1da3eb2c5625 100644 --- a/daemon/containerd/image_list.go +++ b/daemon/containerd/image_list.go @@ -19,7 +19,7 @@ import ( "github.com/moby/buildkit/util/attestation" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -87,7 +87,7 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions) continue } - err := images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc v1.Descriptor) ([]v1.Descriptor, error) { + err := images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { if images.IsIndexType(desc.MediaType) { return images.Children(ctx, contentStore, desc) } @@ -418,7 +418,7 @@ func setupLabelFilter(store content.Store, fltrs filters.Args) (func(image image // processing more content (otherwise it will run for all children). // It will be returned once a matching config is found. errFoundConfig := errors.New("success, found matching config") - err := images.Dispatch(ctx, presentChildrenHandler(store, images.HandlerFunc(func(ctx context.Context, desc v1.Descriptor) (subdescs []v1.Descriptor, err error) { + err := images.Dispatch(ctx, presentChildrenHandler(store, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) { if !images.IsConfigType(desc.MediaType) { return nil, nil } @@ -511,8 +511,8 @@ func computeSharedSize(chainIDs []digest.Digest, layers map[digest.Digest]int, s // getManifestPlatform returns a platform specified by the manifest descriptor // or reads it from its config. -func getManifestPlatform(ctx context.Context, store content.Provider, manifestDesc, configDesc v1.Descriptor) (v1.Platform, error) { - var platform v1.Platform +func getManifestPlatform(ctx context.Context, store content.Provider, manifestDesc, configDesc ocispec.Descriptor) (ocispec.Platform, error) { + var platform ocispec.Platform if manifestDesc.Platform != nil { platform = *manifestDesc.Platform } else { @@ -527,7 +527,7 @@ func getManifestPlatform(ctx context.Context, store content.Provider, manifestDe // isImageManifest returns true if the manifest has no layers or any of its layers is a known image layer. // Some manifests use the image media type for compatibility, even if they are not a real image. -func isImageManifest(mfst v1.Manifest) bool { +func isImageManifest(mfst ocispec.Manifest) bool { if len(mfst.Layers) == 0 { return true } @@ -540,7 +540,7 @@ func isImageManifest(mfst v1.Manifest) bool { } // readConfig reads content pointed by the descriptor and unmarshals it into a specified output. -func readConfig(ctx context.Context, store content.Provider, desc v1.Descriptor, out interface{}) error { +func readConfig(ctx context.Context, store content.Provider, desc ocispec.Descriptor, out interface{}) error { data, err := content.ReadBlob(ctx, store, desc) if err != nil { return errors.Wrapf(err, "failed to read config content") diff --git a/daemon/containerd/image_pull.go b/daemon/containerd/image_pull.go index 160d47cc2f2ac..3e2f623319aba 100644 --- a/daemon/containerd/image_pull.go +++ b/daemon/containerd/image_pull.go @@ -14,13 +14,13 @@ import ( "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/streamformatter" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) // PullImage initiates a pull operation. image is the repository name to pull, and // tagOrDigest may be either empty, or indicate a specific tag or digest to pull. -func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error { var opts []containerd.RemoteOpt if platform != nil { opts = append(opts, containerd.WithPlatform(platforms.Format(*platform))) @@ -49,7 +49,7 @@ func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string, opts = append(opts, containerd.WithResolver(resolver)) jobs := newJobs() - h := images.HandlerFunc(func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) { + h := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { if desc.MediaType != images.MediaTypeDockerSchema1Manifest { jobs.Add(desc) } diff --git a/daemon/containerd/image_snapshot.go b/daemon/containerd/image_snapshot.go index 56fe12c3c361f..35c0816c473c0 100644 --- a/daemon/containerd/image_snapshot.go +++ b/daemon/containerd/image_snapshot.go @@ -7,11 +7,11 @@ import ( "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" "github.com/opencontainers/image-spec/identity" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // PrepareSnapshot prepares a snapshot from a parent image for a container -func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *v1.Platform) error { +func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *ocispec.Platform) error { desc, err := i.resolveDescriptor(ctx, parentImage) if err != nil { return err diff --git a/daemon/create.go b/daemon/create.go index cdd72381fe0fa..a301311be20fc 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -19,7 +19,7 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/runconfig" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/selinux/go-selinux" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -74,7 +74,7 @@ func (daemon *Daemon) containerCreate(ctx context.Context, opts createOpts) (con } if img != nil { p := maximumSpec() - imgPlat := v1.Platform{ + imgPlat := ocispec.Platform{ OS: img.OS, Architecture: img.Architecture, Variant: img.Variant, @@ -117,7 +117,7 @@ func (daemon *Daemon) create(ctx context.Context, opts createOpts) (retC *contai var ( ctr *container.Container img *image.Image - imgManifest *v1.Descriptor + imgManifest *ocispec.Descriptor imgID image.ID err error os = runtime.GOOS @@ -345,7 +345,7 @@ func verifyNetworkingConfig(nwConfig *networktypes.NetworkingConfig) error { } // maximumSpec returns the distribution platform with maximum compatibility for the current node. -func maximumSpec() v1.Platform { +func maximumSpec() ocispec.Platform { p := platforms.DefaultSpec() if p.Architecture == "amd64" { p.Variant = archvariant.AMD64Variant() diff --git a/daemon/image_service.go b/daemon/image_service.go index 8470d18e025d8..4108afeeeb022 100644 --- a/daemon/image_service.go +++ b/daemon/image_service.go @@ -17,7 +17,7 @@ import ( "github.com/docker/docker/layer" "github.com/docker/docker/pkg/archive" "github.com/opencontainers/go-digest" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // ImageService is a temporary interface to assist in the migration to the @@ -26,7 +26,7 @@ import ( type ImageService interface { // Images - PullImage(ctx context.Context, name, tag string, platform *v1.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error + PullImage(ctx context.Context, name, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error PushImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error CreateImage(ctx context.Context, config []byte, parent string, contentStoreDigest digest.Digest) (builder.Image, error) ImageDelete(ctx context.Context, imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) @@ -37,7 +37,7 @@ type ImageService interface { LogImageEvent(imageID, refName, action string) CountImages() int ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) - ImportImage(ctx context.Context, ref reference.Named, platform *v1.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error) + ImportImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error) TagImage(ctx context.Context, imageID image.ID, newTag reference.Named) error GetImage(ctx context.Context, refOrID string, options imagetype.GetImageOpts) (*image.Image, error) ImageHistory(ctx context.Context, name string) ([]*imagetype.HistoryResponseItem, error) @@ -46,8 +46,8 @@ type ImageService interface { // Containerd related methods - PrepareSnapshot(ctx context.Context, id string, image string, platform *v1.Platform) error - GetImageManifest(ctx context.Context, refOrID string, options imagetype.GetImageOpts) (*v1.Descriptor, error) + PrepareSnapshot(ctx context.Context, id string, image string, platform *ocispec.Platform) error + GetImageManifest(ctx context.Context, refOrID string, options imagetype.GetImageOpts) (*ocispec.Descriptor, error) // Layers diff --git a/daemon/images/image.go b/daemon/images/image.go index 739356544cbe4..a773bd0cd2cbc 100644 --- a/daemon/images/image.go +++ b/daemon/images/image.go @@ -17,8 +17,7 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -40,19 +39,19 @@ func (e ErrImageDoesNotExist) Error() string { func (e ErrImageDoesNotExist) NotFound() {} type manifestList struct { - Manifests []specs.Descriptor `json:"manifests"` + Manifests []ocispec.Descriptor `json:"manifests"` } type manifest struct { - Config specs.Descriptor `json:"config"` + Config ocispec.Descriptor `json:"config"` } -func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, image string, platform *v1.Platform) error { +func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, image string, platform *ocispec.Platform) error { // Only makes sense when conatinerd image store is used panic("not implemented") } -func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.Image, platform specs.Platform) (bool, error) { +func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.Image, platform ocispec.Platform) (bool, error) { logger := logrus.WithField("image", img.ID).WithField("desiredPlatform", platforms.Format(platform)) ls, leaseErr := i.leases.ListResources(ctx, leases.Lease{ID: imageKey(img.ID().String())}) @@ -81,7 +80,7 @@ func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.I continue } - ra, err := i.content.ReaderAt(ctx, specs.Descriptor{Digest: digest.Digest(r.ID)}) + ra, err := i.content.ReaderAt(ctx, ocispec.Descriptor{Digest: digest.Digest(r.ID)}) if err != nil { if cerrdefs.IsNotFound(err) { continue @@ -107,12 +106,12 @@ func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.I for _, md := range ml.Manifests { switch md.MediaType { - case specs.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest: + case ocispec.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest: default: continue } - p := specs.Platform{ + p := ocispec.Platform{ Architecture: md.Platform.Architecture, OS: md.Platform.OS, Variant: md.Platform.Variant, @@ -124,7 +123,7 @@ func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.I // Here we have a platform match for the referenced manifest, let's make sure the manifest is actually for the image config we are using. - ra, err := i.content.ReaderAt(ctx, specs.Descriptor{Digest: md.Digest}) + ra, err := i.content.ReaderAt(ctx, ocispec.Descriptor{Digest: md.Digest}) if err != nil { logger.WithField("otherDigest", md.Digest).WithError(err).Error("Could not get reader for manifest") continue @@ -192,7 +191,7 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima return img, nil } -func (i *ImageService) GetImageManifest(ctx context.Context, refOrID string, options imagetypes.GetImageOpts) (*v1.Descriptor, error) { +func (i *ImageService) GetImageManifest(ctx context.Context, refOrID string, options imagetypes.GetImageOpts) (*ocispec.Descriptor, error) { panic("not implemented") } @@ -202,7 +201,7 @@ func (i *ImageService) getImage(ctx context.Context, refOrID string, options ima return } - imgPlat := specs.Platform{ + imgPlat := ocispec.Platform{ OS: retImg.OS, Architecture: retImg.Architecture, Variant: retImg.Variant, @@ -272,16 +271,16 @@ func (i *ImageService) getImage(ctx context.Context, refOrID string, options ima // The reason for this is that CPU variant is not even if the official image config spec as of this writing. // See: https://github.com/opencontainers/image-spec/pull/809 // Since Docker tends to compare platforms from the image config, we need to handle this case. -func OnlyPlatformWithFallback(p specs.Platform) platforms.Matcher { +func OnlyPlatformWithFallback(p ocispec.Platform) platforms.Matcher { return &onlyFallbackMatcher{only: platforms.Only(p), p: platforms.Normalize(p)} } type onlyFallbackMatcher struct { only platforms.Matcher - p specs.Platform + p ocispec.Platform } -func (m *onlyFallbackMatcher) Match(other specs.Platform) bool { +func (m *onlyFallbackMatcher) Match(other ocispec.Platform) bool { if m.only.Match(other) { // It matches, no reason to fallback return true diff --git a/daemon/images/image_builder.go b/daemon/images/image_builder.go index b10878f9f1420..9569651ef99a1 100644 --- a/daemon/images/image_builder.go +++ b/daemon/images/image_builder.go @@ -20,7 +20,7 @@ import ( "github.com/docker/docker/pkg/system" registrypkg "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -149,7 +149,7 @@ func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLay } // TODO: could this use the regular daemon PullImage ? -func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]registry.AuthConfig, output io.Writer, platform *specs.Platform) (*image.Image, error) { +func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]registry.AuthConfig, output io.Writer, platform *ocispec.Platform) (*image.Image, error) { ref, err := reference.ParseNormalizedNamed(name) if err != nil { return nil, err @@ -174,7 +174,7 @@ func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConf img, err := i.GetImage(ctx, name, imagetypes.GetImageOpts{Platform: platform}) if errdefs.IsNotFound(err) && img != nil && platform != nil { - imgPlat := specs.Platform{ + imgPlat := ocispec.Platform{ OS: img.OS, Architecture: img.BaseImgArch(), Variant: img.BaseImgVariant(), diff --git a/daemon/images/image_import.go b/daemon/images/image_import.go index ac0889fe57419..dd32b11a900a7 100644 --- a/daemon/images/image_import.go +++ b/daemon/images/image_import.go @@ -16,7 +16,7 @@ import ( "github.com/docker/docker/layer" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/system" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // ImportImage imports an image, getting the archived layer data from layerReader. @@ -26,7 +26,7 @@ import ( // If the platform is nil, the default host platform is used. // Message is used as the image's history comment. // Image configuration is derived from the dockerfile instructions in changes. -func (i *ImageService) ImportImage(ctx context.Context, newRef reference.Named, platform *specs.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error) { +func (i *ImageService) ImportImage(ctx context.Context, newRef reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error) { if platform == nil { def := platforms.DefaultSpec() platform = &def diff --git a/daemon/images/image_pull.go b/daemon/images/image_pull.go index 154419b3bcd52..b84168544f7e3 100644 --- a/daemon/images/image_pull.go +++ b/daemon/images/image_pull.go @@ -17,14 +17,14 @@ import ( "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/streamformatter" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) // PullImage initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. -func (i *ImageService) PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PullImage(ctx context.Context, image, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error { start := time.Now() // Special case: "pull -a" may send an image name with a // trailing :. This is ugly, but let's not break API @@ -79,7 +79,7 @@ func (i *ImageService) PullImage(ctx context.Context, image, tag string, platfor return nil } -func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error { +func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error { // Include a buffer so that slow client connections don't affect // transfer performance. progressChan := make(chan progress.Progress, 100) diff --git a/daemon/images/images_test.go b/daemon/images/images_test.go index 2608c0b4ed819..24e380d3fdc10 100644 --- a/daemon/images/images_test.go +++ b/daemon/images/images_test.go @@ -3,30 +3,30 @@ package images import ( "testing" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" ) func TestOnlyPlatformWithFallback(t *testing.T) { - p := specs.Platform{ + p := ocispec.Platform{ OS: "linux", Architecture: "arm", Variant: "v8", } // Check no variant - assert.Assert(t, OnlyPlatformWithFallback(p).Match(specs.Platform{ + assert.Assert(t, OnlyPlatformWithFallback(p).Match(ocispec.Platform{ OS: p.OS, Architecture: p.Architecture, })) // check with variant - assert.Assert(t, OnlyPlatformWithFallback(p).Match(specs.Platform{ + assert.Assert(t, OnlyPlatformWithFallback(p).Match(ocispec.Platform{ OS: p.OS, Architecture: p.Architecture, Variant: p.Variant, })) // Make sure non-matches are false. - assert.Assert(t, !OnlyPlatformWithFallback(p).Match(specs.Platform{ + assert.Assert(t, !OnlyPlatformWithFallback(p).Match(ocispec.Platform{ OS: p.OS, Architecture: "amd64", })) diff --git a/daemon/images/store_test.go b/daemon/images/store_test.go index 50f931b9ae377..2732162e7745a 100644 --- a/daemon/images/store_test.go +++ b/daemon/images/store_test.go @@ -14,7 +14,7 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/docker/docker/image" "github.com/opencontainers/go-digest" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "go.etcd.io/bbolt" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -96,7 +96,7 @@ func TestContentStoreForPull(t *testing.T) { } data := []byte(`{}`) - desc := v1.Descriptor{ + desc := ocispec.Descriptor{ Digest: digest.Canonical.FromBytes(data), Size: int64(len(data)), } diff --git a/distribution/config.go b/distribution/config.go index e5048f4de1448..367054e58a141 100644 --- a/distribution/config.go +++ b/distribution/config.go @@ -19,7 +19,7 @@ import ( refstore "github.com/docker/docker/reference" registrypkg "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -60,7 +60,7 @@ type ImagePullConfig struct { // types is used. Schema2Types []string // Platform is the requested platform of the image being pulled - Platform *specs.Platform + Platform *ocispec.Platform } // ImagePushConfig stores push configuration. @@ -141,7 +141,7 @@ func rootFSFromConfig(c []byte) (*image.RootFS, error) { return unmarshalledConfig.RootFS, nil } -func platformFromConfig(c []byte) (*specs.Platform, error) { +func platformFromConfig(c []byte) (*ocispec.Platform, error) { var unmarshalledConfig image.Image if err := json.Unmarshal(c, &unmarshalledConfig); err != nil { return nil, err @@ -154,7 +154,7 @@ func platformFromConfig(c []byte) (*specs.Platform, error) { if !system.IsOSSupported(os) { return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "image operating system %q cannot be used on this platform", os) } - return &specs.Platform{ + return &ocispec.Platform{ OS: os, Architecture: unmarshalledConfig.Architecture, Variant: unmarshalledConfig.Variant, diff --git a/distribution/manifest.go b/distribution/manifest.go index e16a9883dd3d9..621bd1cea1ecd 100644 --- a/distribution/manifest.go +++ b/distribution/manifest.go @@ -18,7 +18,7 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -97,7 +97,7 @@ func hasDistributionSource(label, repo string) bool { return false } -func (m *manifestStore) getLocal(ctx context.Context, desc specs.Descriptor, ref reference.Named) (distribution.Manifest, error) { +func (m *manifestStore) getLocal(ctx context.Context, desc ocispec.Descriptor, ref reference.Named) (distribution.Manifest, error) { ra, err := m.local.ReaderAt(ctx, desc) if err != nil { return nil, errors.Wrap(err, "error getting content store reader") @@ -153,7 +153,7 @@ func (m *manifestStore) getLocal(ctx context.Context, desc specs.Descriptor, ref return manifest, nil } -func (m *manifestStore) getMediaType(ctx context.Context, desc specs.Descriptor) (string, error) { +func (m *manifestStore) getMediaType(ctx context.Context, desc ocispec.Descriptor) (string, error) { ra, err := m.local.ReaderAt(ctx, desc) if err != nil { return "", errors.Wrap(err, "error getting reader to detect media type") @@ -167,7 +167,7 @@ func (m *manifestStore) getMediaType(ctx context.Context, desc specs.Descriptor) return mt, nil } -func (m *manifestStore) Get(ctx context.Context, desc specs.Descriptor, ref reference.Named) (distribution.Manifest, error) { +func (m *manifestStore) Get(ctx context.Context, desc ocispec.Descriptor, ref reference.Named) (distribution.Manifest, error) { l := log.G(ctx) if desc.MediaType == "" { @@ -227,7 +227,7 @@ func (m *manifestStore) Get(ctx context.Context, desc specs.Descriptor, ref refe return manifest, nil } -func (m *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, desc specs.Descriptor, w content.Writer, ref reference.Named) error { +func (m *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, desc ocispec.Descriptor, w content.Writer, ref reference.Named) error { mt, payload, err := manifest.Payload() if err != nil { return err @@ -282,12 +282,12 @@ func detectManifestBlobMediaType(dt []byte) (string, error) { // So pretty much if we don't have a media type we can fall back to OCI. // This does have a special fallback for schema1 manifests just because it is easy to detect. switch mfst.MediaType { - case schema2.MediaTypeManifest, specs.MediaTypeImageManifest: + case schema2.MediaTypeManifest, ocispec.MediaTypeImageManifest: if mfst.Manifests != nil || mfst.FSLayers != nil { return "", fmt.Errorf(`media-type: %q should not have "manifests" or "fsLayers"`, mfst.MediaType) } return mfst.MediaType, nil - case manifestlist.MediaTypeManifestList, specs.MediaTypeImageIndex: + case manifestlist.MediaTypeManifestList, ocispec.MediaTypeImageIndex: if mfst.Config != nil || mfst.Layers != nil || mfst.FSLayers != nil { return "", fmt.Errorf(`media-type: %q should not have "config", "layers", or "fsLayers"`, mfst.MediaType) } @@ -307,10 +307,10 @@ func detectManifestBlobMediaType(dt []byte) (string, error) { return schema1.MediaTypeManifest, nil case mfst.Config != nil && mfst.Manifests == nil && mfst.FSLayers == nil, mfst.Layers != nil && mfst.Manifests == nil && mfst.FSLayers == nil: - return specs.MediaTypeImageManifest, nil + return ocispec.MediaTypeImageManifest, nil case mfst.Config == nil && mfst.Layers == nil && mfst.FSLayers == nil: // fallback to index - return specs.MediaTypeImageIndex, nil + return ocispec.MediaTypeImageIndex, nil } return "", errors.New("media-type: cannot determine") } diff --git a/distribution/manifest_test.go b/distribution/manifest_test.go index e5410b1724ba2..73e752bd7ae4d 100644 --- a/distribution/manifest_test.go +++ b/distribution/manifest_test.go @@ -20,7 +20,7 @@ import ( "github.com/docker/distribution/reference" "github.com/google/go-cmp/cmp/cmpopts" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "gotest.tools/v3/assert" "gotest.tools/v3/assert/cmp" @@ -128,12 +128,12 @@ func (w *testingContentWriterWrapper) Commit(ctx context.Context, size int64, dg } func TestManifestStore(t *testing.T) { - ociManifest := &specs.Manifest{} + ociManifest := &ocispec.Manifest{} serialized, err := json.Marshal(ociManifest) assert.NilError(t, err) dgst := digest.Canonical.FromBytes(serialized) - setupTest := func(t *testing.T) (reference.Named, specs.Descriptor, *mockManifestGetter, *manifestStore, content.Store, func(*testing.T)) { + setupTest := func(t *testing.T) (reference.Named, ocispec.Descriptor, *mockManifestGetter, *manifestStore, content.Store, func(*testing.T)) { root, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", "_")) assert.NilError(t, err) defer func() { @@ -147,7 +147,7 @@ func TestManifestStore(t *testing.T) { mg := &mockManifestGetter{manifests: make(map[digest.Digest]distribution.Manifest)} store := &manifestStore{local: cs, remote: mg} - desc := specs.Descriptor{Digest: dgst, MediaType: specs.MediaTypeImageManifest, Size: int64(len(serialized))} + desc := ocispec.Descriptor{Digest: dgst, MediaType: ocispec.MediaTypeImageManifest, Size: int64(len(serialized))} ref, err := reference.Parse("foo/bar") assert.NilError(t, err) @@ -159,10 +159,10 @@ func TestManifestStore(t *testing.T) { ctx := context.Background() - m, _, err := distribution.UnmarshalManifest(specs.MediaTypeImageManifest, serialized) + m, _, err := distribution.UnmarshalManifest(ocispec.MediaTypeImageManifest, serialized) assert.NilError(t, err) - writeManifest := func(t *testing.T, cs ContentStore, desc specs.Descriptor, opts ...content.Opt) { + writeManifest := func(t *testing.T, cs ContentStore, desc ocispec.Descriptor, opts ...content.Opt) { ingestKey := remotes.MakeRefKey(ctx, desc) w, err := cs.Writer(ctx, content.WithDescriptor(desc), content.WithRef(ingestKey)) assert.NilError(t, err) @@ -185,7 +185,7 @@ func TestManifestStore(t *testing.T) { } // All tests should end up with no active ingest - checkIngest := func(t *testing.T, cs content.Store, desc specs.Descriptor) { + checkIngest := func(t *testing.T, cs content.Store, desc ocispec.Descriptor) { ingestKey := remotes.MakeRefKey(ctx, desc) _, err := cs.Status(ctx, ingestKey) assert.Check(t, cerrdefs.IsNotFound(err), err) @@ -354,9 +354,9 @@ func TestDetectManifestBlobMediaType(t *testing.T) { } cases := map[string]testCase{ "mediaType is set": {[]byte(`{"mediaType": "bananas"}`), "bananas"}, - "oci manifest": {[]byte(`{"config": {}}`), specs.MediaTypeImageManifest}, + "oci manifest": {[]byte(`{"config": {}}`), ocispec.MediaTypeImageManifest}, "schema1": {[]byte(`{"fsLayers": []}`), schema1.MediaTypeManifest}, - "oci index fallback": {[]byte(`{}`), specs.MediaTypeImageIndex}, + "oci index fallback": {[]byte(`{}`), ocispec.MediaTypeImageIndex}, // Make sure we prefer mediaType "mediaType and config set": {[]byte(`{"mediaType": "bananas", "config": {}}`), "bananas"}, "mediaType and fsLayers set": {[]byte(`{"mediaType": "bananas", "fsLayers": []}`), "bananas"}, @@ -394,7 +394,7 @@ func TestDetectManifestBlobMediaTypeInvalid(t *testing.T) { `media-type: "application/vnd.docker.distribution.manifest.v2+json" should not have "manifests" or "fsLayers"`, }, "oci manifest mediaType with manifests": { - []byte(`{"mediaType": "` + specs.MediaTypeImageManifest + `","manifests":[]}`), + []byte(`{"mediaType": "` + ocispec.MediaTypeImageManifest + `","manifests":[]}`), `media-type: "application/vnd.oci.image.manifest.v1+json" should not have "manifests" or "fsLayers"`, }, "manifest list mediaType with fsLayers": { @@ -402,11 +402,11 @@ func TestDetectManifestBlobMediaTypeInvalid(t *testing.T) { `media-type: "application/vnd.docker.distribution.manifest.list.v2+json" should not have "config", "layers", or "fsLayers"`, }, "index mediaType with layers": { - []byte(`{"mediaType": "` + specs.MediaTypeImageIndex + `","layers":[]}`), + []byte(`{"mediaType": "` + ocispec.MediaTypeImageIndex + `","layers":[]}`), `media-type: "application/vnd.oci.image.index.v1+json" should not have "config", "layers", or "fsLayers"`, }, "index mediaType with config": { - []byte(`{"mediaType": "` + specs.MediaTypeImageIndex + `","config":{}}`), + []byte(`{"mediaType": "` + ocispec.MediaTypeImageIndex + `","config":{}}`), `media-type: "application/vnd.oci.image.index.v1+json" should not have "config", "layers", or "fsLayers"`, }, "config and manifests": { diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index fdb2672d08d12..06e10a9b8d54b 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -31,7 +31,7 @@ import ( refstore "github.com/docker/docker/reference" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" archvariant "github.com/tonistiigi/go-archvariant" @@ -348,7 +348,7 @@ func (ld *layerDescriptor) Registered(diffID layer.DiffID) { _ = ld.metadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.Name.Name()}) } -func (p *puller) pullTag(ctx context.Context, ref reference.Named, platform *specs.Platform) (tagUpdated bool, err error) { +func (p *puller) pullTag(ctx context.Context, ref reference.Named, platform *ocispec.Platform) (tagUpdated bool, err error) { var ( tagOrDigest string // Used for logging/progress only dgst digest.Digest @@ -381,7 +381,7 @@ func (p *puller) pullTag(ctx context.Context, ref reference.Named, platform *spe "remote": ref, })) - desc := specs.Descriptor{ + desc := ocispec.Descriptor{ MediaType: mt, Digest: dgst, Size: size, @@ -519,7 +519,7 @@ func (p *puller) validateMediaType(mediaType string) error { return invalidManifestClassError{mediaType, configClass} } -func (p *puller) pullSchema1(ctx context.Context, ref reference.Reference, unverifiedManifest *schema1.SignedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { +func (p *puller) pullSchema1(ctx context.Context, ref reference.Reference, unverifiedManifest *schema1.SignedManifest, platform *ocispec.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { if platform != nil { // Early bath if the requested OS doesn't match that of the configuration. // This avoids doing the download, only to potentially fail later. @@ -616,7 +616,7 @@ func checkSupportedMediaType(mediaType string) error { return unsupportedMediaTypeError{MediaType: mediaType} } -func (p *puller) pullSchema2Layers(ctx context.Context, target distribution.Descriptor, layers []distribution.Descriptor, platform *specs.Platform) (id digest.Digest, err error) { +func (p *puller) pullSchema2Layers(ctx context.Context, target distribution.Descriptor, layers []distribution.Descriptor, platform *ocispec.Platform) (id digest.Digest, err error) { if _, err := p.config.ImageStore.Get(ctx, target.Digest); err == nil { // If the image already exists locally, no need to pull // anything. @@ -669,11 +669,11 @@ func (p *puller) pullSchema2Layers(ctx context.Context, target distribution.Desc }() var ( - configJSON []byte // raw serialized image config - downloadedRootFS *image.RootFS // rootFS from registered layers - configRootFS *image.RootFS // rootFS from configuration - release func() // release resources from rootFS download - configPlatform *specs.Platform // for LCOW when registering downloaded layers + configJSON []byte // raw serialized image config + downloadedRootFS *image.RootFS // rootFS from registered layers + configRootFS *image.RootFS // rootFS from configuration + release func() // release resources from rootFS download + configPlatform *ocispec.Platform // for LCOW when registering downloaded layers ) layerStoreOS := runtime.GOOS @@ -798,7 +798,7 @@ func (p *puller) pullSchema2Layers(ctx context.Context, target distribution.Desc return imageID, nil } -func (p *puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { +func (p *puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *ocispec.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { manifestDigest, err = schema2ManifestDigest(ref, mfst) if err != nil { return "", "", err @@ -807,7 +807,7 @@ func (p *puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *sch return id, manifestDigest, err } -func (p *puller) pullOCI(ctx context.Context, ref reference.Named, mfst *ocischema.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { +func (p *puller) pullOCI(ctx context.Context, ref reference.Named, mfst *ocischema.DeserializedManifest, platform *ocispec.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { manifestDigest, err = schema2ManifestDigest(ref, mfst) if err != nil { return "", "", err @@ -816,7 +816,7 @@ func (p *puller) pullOCI(ctx context.Context, ref reference.Named, mfst *ocische return id, manifestDigest, err } -func receiveConfig(configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, *specs.Platform, error) { +func receiveConfig(configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, *ocispec.Platform, error) { select { case configJSON := <-configChan: rootfs, err := rootFSFromConfig(configJSON) @@ -837,13 +837,13 @@ func receiveConfig(configChan <-chan []byte, errChan <-chan error) ([]byte, *ima // pullManifestList handles "manifest lists" which point to various // platform-specific manifests. -func (p *puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, pp *specs.Platform) (id digest.Digest, manifestListDigest digest.Digest, err error) { +func (p *puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, pp *ocispec.Platform) (id digest.Digest, manifestListDigest digest.Digest, err error) { manifestListDigest, err = schema2ManifestDigest(ref, mfstList) if err != nil { return "", "", err } - var platform specs.Platform + var platform ocispec.Platform if pp != nil { platform = *pp } @@ -856,7 +856,7 @@ func (p *puller) pullManifestList(ctx context.Context, ref reference.Named, mfst return "", "", err } - desc := specs.Descriptor{ + desc := ocispec.Descriptor{ Digest: match.Digest, Size: match.Size, MediaType: match.MediaType, @@ -942,7 +942,7 @@ func (p *puller) pullSchema2Config(ctx context.Context, dgst digest.Digest) (con } type noMatchesErr struct { - platform specs.Platform + platform ocispec.Platform } func (e noMatchesErr) Error() string { @@ -1081,13 +1081,13 @@ func createDownloadFile() (*os.File, error) { return os.CreateTemp("", "GetImageBlob") } -func toOCIPlatform(p manifestlist.PlatformSpec) *specs.Platform { +func toOCIPlatform(p manifestlist.PlatformSpec) *ocispec.Platform { // distribution pkg does define platform as pointer so this hack for empty struct // is necessary. This is temporary until correct OCI image-spec package is used. if p.OS == "" && p.Architecture == "" && p.Variant == "" && p.OSVersion == "" && p.OSFeatures == nil && p.Features == nil { return nil } - return &specs.Platform{ + return &ocispec.Platform{ OS: p.OS, Architecture: p.Architecture, Variant: p.Variant, @@ -1097,7 +1097,7 @@ func toOCIPlatform(p manifestlist.PlatformSpec) *specs.Platform { } // maximumSpec returns the distribution platform with maximum compatibility for the current node. -func maximumSpec() specs.Platform { +func maximumSpec() ocispec.Platform { p := platforms.DefaultSpec() if p.Architecture == "amd64" { p.Variant = archvariant.AMD64Variant() diff --git a/distribution/pull_v2_test.go b/distribution/pull_v2_test.go index 381590c4ac2d3..ca23d3b0dd7ae 100644 --- a/distribution/pull_v2_test.go +++ b/distribution/pull_v2_test.go @@ -20,7 +20,7 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" ) @@ -194,7 +194,7 @@ func TestValidateManifest(t *testing.T) { } func TestFormatPlatform(t *testing.T) { - var platform specs.Platform + var platform ocispec.Platform var result = formatPlatform(platform) if strings.HasPrefix(result, "unknown") { t.Fatal("expected formatPlatform to show a known platform") diff --git a/distribution/pull_v2_unix.go b/distribution/pull_v2_unix.go index 6d83253f528ac..1c1524bd15af9 100644 --- a/distribution/pull_v2_unix.go +++ b/distribution/pull_v2_unix.go @@ -10,7 +10,7 @@ import ( "github.com/containerd/containerd/platforms" "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) @@ -19,7 +19,7 @@ func (ld *layerDescriptor) open(ctx context.Context) (distribution.ReadSeekClose return blobs.Open(ctx, ld.digest) } -func filterManifests(manifests []manifestlist.ManifestDescriptor, p specs.Platform) []manifestlist.ManifestDescriptor { +func filterManifests(manifests []manifestlist.ManifestDescriptor, p ocispec.Platform) []manifestlist.ManifestDescriptor { p = platforms.Normalize(withDefault(p)) m := platforms.Only(p) var matches []manifestlist.ManifestDescriptor @@ -53,7 +53,7 @@ func checkImageCompatibility(imageOS, imageOSVersion string) error { return nil } -func withDefault(p specs.Platform) specs.Platform { +func withDefault(p ocispec.Platform) ocispec.Platform { def := maximumSpec() if p.OS == "" { p.OS = def.OS @@ -65,7 +65,7 @@ func withDefault(p specs.Platform) specs.Platform { return p } -func formatPlatform(platform specs.Platform) string { +func formatPlatform(platform ocispec.Platform) string { if platform.OS == "" { platform = platforms.DefaultSpec() } diff --git a/distribution/pull_v2_windows.go b/distribution/pull_v2_windows.go index acd67feb596b0..9e7dc25cf316e 100644 --- a/distribution/pull_v2_windows.go +++ b/distribution/pull_v2_windows.go @@ -18,7 +18,7 @@ import ( "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/pkg/system" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) @@ -65,7 +65,7 @@ func (ld *layerDescriptor) open(ctx context.Context) (distribution.ReadSeekClose return rsc, err } -func filterManifests(manifests []manifestlist.ManifestDescriptor, p specs.Platform) []manifestlist.ManifestDescriptor { +func filterManifests(manifests []manifestlist.ManifestDescriptor, p ocispec.Platform) []manifestlist.ManifestDescriptor { version := osversion.Get() osVersion := fmt.Sprintf("%d.%d.%d", version.MajorVersion, version.MinorVersion, version.Build) logrus.Debugf("will prefer Windows entries with version %s", osVersion) @@ -139,7 +139,7 @@ func checkImageCompatibility(imageOS, imageOSVersion string) error { return nil } -func formatPlatform(platform specs.Platform) string { +func formatPlatform(platform ocispec.Platform) string { if platform.OS == "" { platform = platforms.DefaultSpec() } diff --git a/integration/container/create_test.go b/integration/container/create_test.go index 6f0832cf14dd4..eabb2a69b51d1 100644 --- a/integration/container/create_test.go +++ b/integration/container/create_test.go @@ -17,7 +17,7 @@ import ( "github.com/docker/docker/errdefs" ctr "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/oci" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" "gotest.tools/v3/poll" @@ -475,7 +475,7 @@ func TestCreateDifferentPlatform(t *testing.T) { assert.Assert(t, img.Architecture != "") t.Run("different os", func(t *testing.T) { - p := specs.Platform{ + p := ocispec.Platform{ OS: img.Os + "DifferentOS", Architecture: img.Architecture, Variant: img.Variant, @@ -484,7 +484,7 @@ func TestCreateDifferentPlatform(t *testing.T) { assert.Assert(t, client.IsErrNotFound(err), err) }) t.Run("different cpu arch", func(t *testing.T) { - p := specs.Platform{ + p := ocispec.Platform{ OS: img.Os, Architecture: img.Architecture + "DifferentArch", Variant: img.Variant, diff --git a/integration/image/pull_test.go b/integration/image/pull_test.go index bf63045a616dd..a0ecb78d63360 100644 --- a/integration/image/pull_test.go +++ b/integration/image/pull_test.go @@ -19,7 +19,7 @@ import ( "github.com/docker/docker/testutil/registry" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" - imagespec "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" "gotest.tools/v3/skip" ) @@ -36,7 +36,7 @@ func TestImagePullPlatformInvalid(t *testing.T) { assert.Assert(t, errdefs.IsInvalidParameter(err)) } -func createTestImage(ctx context.Context, t testing.TB, store content.Store) imagespec.Descriptor { +func createTestImage(ctx context.Context, t testing.TB, store content.Store) ocispec.Descriptor { w, err := store.Writer(ctx, content.WithRef("layer")) assert.NilError(t, err) defer w.Close() @@ -55,11 +55,11 @@ func createTestImage(ctx context.Context, t testing.TB, store content.Store) ima platform := platforms.DefaultSpec() - img := imagespec.Image{ + img := ocispec.Image{ Architecture: platform.Architecture, OS: platform.OS, - RootFS: imagespec.RootFS{Type: "layers", DiffIDs: []digest.Digest{layerDigest}}, - Config: imagespec.ImageConfig{WorkingDir: "/"}, + RootFS: ocispec.RootFS{Type: "layers", DiffIDs: []digest.Digest{layerDigest}}, + Config: ocispec.ImageConfig{WorkingDir: "/"}, } imgJSON, err := json.Marshal(img) assert.NilError(t, err) @@ -77,17 +77,17 @@ func createTestImage(ctx context.Context, t testing.TB, store content.Store) ima info, err := store.Info(ctx, layerDigest) assert.NilError(t, err) - manifest := imagespec.Manifest{ + manifest := ocispec.Manifest{ Versioned: specs.Versioned{ SchemaVersion: 2, }, MediaType: images.MediaTypeDockerSchema2Manifest, - Config: imagespec.Descriptor{ + Config: ocispec.Descriptor{ MediaType: images.MediaTypeDockerSchema2Config, Digest: configDigest, Size: int64(len(imgJSON)), }, - Layers: []imagespec.Descriptor{{ + Layers: []ocispec.Descriptor{{ MediaType: images.MediaTypeDockerSchema2Layer, Digest: layerDigest, Size: info.Size, @@ -107,7 +107,7 @@ func createTestImage(ctx context.Context, t testing.TB, store content.Store) ima manifestDigest := w.Digest() w.Close() - return imagespec.Descriptor{ + return ocispec.Descriptor{ MediaType: images.MediaTypeDockerSchema2Manifest, Digest: manifestDigest, Size: int64(len(manifestJSON)), diff --git a/integration/internal/container/container.go b/integration/internal/container/container.go index 6559bd4f4a719..2b45be9abd041 100644 --- a/integration/internal/container/container.go +++ b/integration/internal/container/container.go @@ -9,7 +9,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" ) @@ -20,7 +20,7 @@ type TestContainerConfig struct { Config *container.Config HostConfig *container.HostConfig NetworkingConfig *network.NetworkingConfig - Platform *specs.Platform + Platform *ocispec.Platform } // create creates a container with the specified options diff --git a/integration/internal/container/ops.go b/integration/internal/container/ops.go index 18c93ea0ab956..33d977700623c 100644 --- a/integration/internal/container/ops.go +++ b/integration/internal/container/ops.go @@ -8,7 +8,7 @@ import ( networktypes "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/strslice" "github.com/docker/go-connections/nat" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // WithName sets the name of the container @@ -204,7 +204,7 @@ func WithExtraHost(extraHost string) func(*TestContainerConfig) { } // WithPlatform specifies the desired platform the image should have. -func WithPlatform(p *specs.Platform) func(*TestContainerConfig) { +func WithPlatform(p *ocispec.Platform) func(*TestContainerConfig) { return func(c *TestContainerConfig) { c.Platform = p } diff --git a/integration/plugin/common/plugin_test.go b/integration/plugin/common/plugin_test.go index 3c6670a5eea13..630d4a09c3455 100644 --- a/integration/plugin/common/plugin_test.go +++ b/integration/plugin/common/plugin_test.go @@ -23,7 +23,7 @@ import ( "github.com/docker/docker/testutil/fixtures/plugin" "github.com/docker/docker/testutil/registry" "github.com/docker/docker/testutil/request" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" "gotest.tools/v3/skip" @@ -306,7 +306,7 @@ func TestPluginBackCompatMediaTypes(t *testing.T) { assert.NilError(t, err) defer rdr.Close() - var m v1.Manifest + var m ocispec.Manifest assert.NilError(t, json.NewDecoder(rdr).Decode(&m)) assert.Check(t, is.Equal(m.MediaType, images.MediaTypeDockerSchema2Manifest)) assert.Check(t, is.Len(m.Layers, 1)) diff --git a/libcontainerd/remote/client.go b/libcontainerd/remote/client.go index 837f770e3cd7b..6ea98b0c9c993 100644 --- a/libcontainerd/remote/client.go +++ b/libcontainerd/remote/client.go @@ -29,7 +29,7 @@ import ( libcontainerdtypes "github.com/docker/docker/libcontainerd/types" "github.com/docker/docker/pkg/ioutils" "github.com/hashicorp/go-multierror" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -448,12 +448,12 @@ func (t *task) CreateCheckpoint(ctx context.Context, checkpointDir string, exit if err != nil { return errdefs.System(errors.Wrapf(err, "failed to retrieve checkpoint data")) } - var index v1.Index + var index ocispec.Index if err := json.Unmarshal(b, &index); err != nil { return errdefs.System(errors.Wrapf(err, "failed to decode checkpoint data")) } - var cpDesc *v1.Descriptor + var cpDesc *ocispec.Descriptor for _, m := range index.Manifests { m := m if m.MediaType == images.MediaTypeContainerd1Checkpoint { diff --git a/plugin/backend_linux.go b/plugin/backend_linux.go index 681a3624a2e92..cdc2831a1ff56 100644 --- a/plugin/backend_linux.go +++ b/plugin/backend_linux.go @@ -35,7 +35,7 @@ import ( v2 "github.com/docker/docker/plugin/v2" "github.com/moby/sys/mount" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -165,19 +165,19 @@ func (pm *Manager) Privileges(ctx context.Context, ref reference.Named, metaHead configSeen bool ) - h := func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) { + h := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch desc.MediaType { - case schema2.MediaTypeManifest, specs.MediaTypeImageManifest: + case schema2.MediaTypeManifest, ocispec.MediaTypeImageManifest: data, err := content.ReadBlob(ctx, pm.blobStore, desc) if err != nil { return nil, errors.Wrapf(err, "error reading image manifest from blob store for %s", ref) } - var m specs.Manifest + var m ocispec.Manifest if err := json.Unmarshal(data, &m); err != nil { return nil, errors.Wrapf(err, "error unmarshaling image manifest for %s", ref) } - return []specs.Descriptor{m.Config}, nil + return []ocispec.Descriptor{m.Config}, nil case schema2.MediaTypePluginConfig: configSeen = true data, err := content.ReadBlob(ctx, pm.blobStore, desc) @@ -383,7 +383,7 @@ func (pm *Manager) Push(ctx context.Context, name string, metaHeader http.Header out, waitProgress := setupProgressOutput(outStream, cancel) defer waitProgress() - progressHandler := images.HandlerFunc(func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) { + progressHandler := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { logrus.WithField("mediaType", desc.MediaType).WithField("digest", desc.Digest.String()).Debug("Preparing to push plugin layer") id := stringid.TruncateID(desc.Digest.String()) pj.add(remotes.MakeRefKey(ctx, desc), id) @@ -469,7 +469,7 @@ func (pm *Manager) Push(ctx context.Context, name string, metaHeader http.Header // even though this is set on the descriptor // The OCI types do not have this field. type manifest struct { - specs.Manifest + ocispec.Manifest MediaType string `json:"mediaType,omitempty"` } @@ -482,7 +482,7 @@ func buildManifest(ctx context.Context, s content.Manager, config digest.Digest, if err != nil { return m, errors.Wrapf(err, "error reading plugin config content for digest %s", config) } - m.Config = specs.Descriptor{ + m.Config = ocispec.Descriptor{ MediaType: mediaTypePluginConfig, Size: configInfo.Size, Digest: configInfo.Digest, @@ -493,7 +493,7 @@ func buildManifest(ctx context.Context, s content.Manager, config digest.Digest, if err != nil { return m, errors.Wrapf(err, "error fetching info for content digest %s", l) } - m.Layers = append(m.Layers, specs.Descriptor{ + m.Layers = append(m.Layers, ocispec.Descriptor{ MediaType: images.MediaTypeDockerSchema2LayerGzip, // TODO: This is assuming everything is a gzip compressed layer, but that may not be true. Digest: l, Size: info.Size, @@ -504,12 +504,12 @@ func buildManifest(ctx context.Context, s content.Manager, config digest.Digest, // getManifestDescriptor gets the OCI descriptor for a manifest // It will generate a manifest if one does not exist -func (pm *Manager) getManifestDescriptor(ctx context.Context, p *v2.Plugin) (specs.Descriptor, error) { +func (pm *Manager) getManifestDescriptor(ctx context.Context, p *v2.Plugin) (ocispec.Descriptor, error) { logger := logrus.WithField("plugin", p.Name()).WithField("digest", p.Manifest) if p.Manifest != "" { info, err := pm.blobStore.Info(ctx, p.Manifest) if err == nil { - desc := specs.Descriptor{ + desc := ocispec.Descriptor{ Size: info.Size, Digest: info.Digest, MediaType: images.MediaTypeDockerSchema2Manifest, @@ -524,7 +524,7 @@ func (pm *Manager) getManifestDescriptor(ctx context.Context, p *v2.Plugin) (spe manifest, err := buildManifest(ctx, pm.blobStore, p.Config, p.Blobsums) if err != nil { - return specs.Descriptor{}, err + return ocispec.Descriptor{}, err } desc, err := writeManifest(ctx, pm.blobStore, &manifest) @@ -538,9 +538,9 @@ func (pm *Manager) getManifestDescriptor(ctx context.Context, p *v2.Plugin) (spe return desc, nil } -func writeManifest(ctx context.Context, cs content.Store, m *manifest) (specs.Descriptor, error) { +func writeManifest(ctx context.Context, cs content.Store, m *manifest) (ocispec.Descriptor, error) { platform := platforms.DefaultSpec() - desc := specs.Descriptor{ + desc := ocispec.Descriptor{ MediaType: images.MediaTypeDockerSchema2Manifest, Platform: &platform, } diff --git a/plugin/fetch_linux.go b/plugin/fetch_linux.go index c33cdc200c548..bfad62c315020 100644 --- a/plugin/fetch_linux.go +++ b/plugin/fetch_linux.go @@ -19,7 +19,7 @@ import ( "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/stringid" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -88,8 +88,8 @@ func (pm *Manager) fetch(ctx context.Context, ref reference.Named, auth *registr headers := http.Header{} headers.Add("Accept", images.MediaTypeDockerSchema2Manifest) headers.Add("Accept", images.MediaTypeDockerSchema2ManifestList) - headers.Add("Accept", specs.MediaTypeImageManifest) - headers.Add("Accept", specs.MediaTypeImageIndex) + headers.Add("Accept", ocispec.MediaTypeImageManifest) + headers.Add("Accept", ocispec.MediaTypeImageIndex) resolver, _ = pm.newResolver(ctx, nil, auth, headers, false) if resolver != nil { resolved, desc, err = resolver.Resolve(ctx, withDomain.String()) @@ -118,12 +118,12 @@ func (pm *Manager) fetch(ctx context.Context, ref reference.Named, auth *registr // if there are multiple layers to fetch we may end up extracting layers in the wrong // order. func applyLayer(cs content.Store, dir string, out progress.Output) images.HandlerFunc { - return func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) { + return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch desc.MediaType { case - specs.MediaTypeImageLayer, + ocispec.MediaTypeImageLayer, images.MediaTypeDockerSchema2Layer, - specs.MediaTypeImageLayerGzip, + ocispec.MediaTypeImageLayerGzip, images.MediaTypeDockerSchema2LayerGzip: default: return nil, nil @@ -150,7 +150,7 @@ func applyLayer(cs content.Store, dir string, out progress.Output) images.Handle func childrenHandler(cs content.Store) images.HandlerFunc { ch := images.ChildrenHandler(cs) - return func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) { + return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch desc.MediaType { case mediaTypePluginConfig: return nil, nil @@ -167,15 +167,15 @@ type fetchMeta struct { } func storeFetchMetadata(m *fetchMeta) images.HandlerFunc { - return func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) { + return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch desc.MediaType { case images.MediaTypeDockerSchema2LayerForeignGzip, images.MediaTypeDockerSchema2Layer, - specs.MediaTypeImageLayer, - specs.MediaTypeImageLayerGzip: + ocispec.MediaTypeImageLayer, + ocispec.MediaTypeImageLayerGzip: m.blobs = append(m.blobs, desc.Digest) - case specs.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest: + case ocispec.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest: m.manifest = desc.Digest case mediaTypePluginConfig: m.config = desc.Digest @@ -196,9 +196,9 @@ func validateFetchedMetadata(md fetchMeta) error { // withFetchProgress is a fetch handler which registers a descriptor with a progress func withFetchProgress(cs content.Store, out progress.Output, ref reference.Named) images.HandlerFunc { - return func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) { + return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { switch desc.MediaType { - case specs.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest: + case ocispec.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest: tn := reference.TagNameOnly(ref) tagged := tn.(reference.Tagged) progress.Messagef(out, tagged.Tag(), "Pulling from %s", reference.FamiliarName(ref)) @@ -207,8 +207,8 @@ func withFetchProgress(cs content.Store, out progress.Output, ref reference.Name case images.MediaTypeDockerSchema2LayerGzip, images.MediaTypeDockerSchema2Layer, - specs.MediaTypeImageLayer, - specs.MediaTypeImageLayerGzip: + ocispec.MediaTypeImageLayer, + ocispec.MediaTypeImageLayerGzip: default: return nil, nil } diff --git a/plugin/manager_linux.go b/plugin/manager_linux.go index 5ffe4a152fd17..72eda2d4b4f7a 100644 --- a/plugin/manager_linux.go +++ b/plugin/manager_linux.go @@ -18,7 +18,7 @@ import ( v2 "github.com/docker/docker/plugin/v2" "github.com/moby/sys/mount" "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -267,7 +267,7 @@ func (pm *Manager) upgradePlugin(p *v2.Plugin, configDigest, manifestDigest dige } func (pm *Manager) setupNewPlugin(configDigest digest.Digest, privileges *types.PluginPrivileges) (types.PluginConfig, error) { - configRA, err := pm.blobStore.ReaderAt(context.TODO(), specs.Descriptor{Digest: configDigest}) + configRA, err := pm.blobStore.ReaderAt(context.TODO(), ocispec.Descriptor{Digest: configDigest}) if err != nil { return types.PluginConfig{}, err } From 4c6b8e737fd13cf000a1714560588fa68fca3633 Mon Sep 17 00:00:00 2001 From: Jeyanthinath Muthuram Date: Mon, 8 May 2023 19:03:44 +0530 Subject: [PATCH 053/293] added alias validation Signed-off-by: Jeyanthinath Muthuram (cherry picked from commit 71d7908656ab3d4d0999578b3367025d1acc1f77) Signed-off-by: Sebastiaan van Stijn --- hack/validate/golangci-lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/validate/golangci-lint.yml b/hack/validate/golangci-lint.yml index 618fd60cf3154..ab188068506df 100644 --- a/hack/validate/golangci-lint.yml +++ b/hack/validate/golangci-lint.yml @@ -34,6 +34,8 @@ linters-settings: # own errdefs package (or vice-versa). - pkg: github.com/containerd/containerd/errdefs alias: cerrdefs + - pkg: github.com/opencontainers/image-spec/specs-go/v1 + alias: ocispec govet: check-shadowing: false From 4217d9ea0ad87e221a64f1806cbdccb8b234f2bf Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 31 May 2023 11:52:18 +0200 Subject: [PATCH 054/293] Dockerfile: use COPY --link to copy artifacts from build-stages Build-cache for the build-stages themselves are already invalidated if the base-images they're using is updated, and the COPY operations don't depend on previous steps (as there's no overlap between artifacts copied). Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 637ca59375cdc26486e9451eea1aa6a9aac6154e) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf73d8525ef25..56350d6d0feb6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -439,30 +439,30 @@ FROM containerutil-windows-${TARGETARCH} AS containerutil-windows FROM containerutil-${TARGETOS} AS containerutil FROM base AS dev-systemd-false -COPY --from=dockercli /build/ /usr/local/cli -COPY --from=frozen-images /build/ /docker-frozen-images -COPY --from=swagger /build/ /usr/local/bin/ -COPY --from=delve /build/ /usr/local/bin/ -COPY --from=tomll /build/ /usr/local/bin/ -COPY --from=gowinres /build/ /usr/local/bin/ -COPY --from=tini /build/ /usr/local/bin/ -COPY --from=registry /build/ /usr/local/bin/ +COPY --link --from=dockercli /build/ /usr/local/cli +COPY --link --from=frozen-images /build/ /docker-frozen-images +COPY --link --from=swagger /build/ /usr/local/bin/ +COPY --link --from=delve /build/ /usr/local/bin/ +COPY --link --from=tomll /build/ /usr/local/bin/ +COPY --link --from=gowinres /build/ /usr/local/bin/ +COPY --link --from=tini /build/ /usr/local/bin/ +COPY --link --from=registry /build/ /usr/local/bin/ # Skip the CRIU stage for now, as the opensuse package repository is sometimes # unstable, and we're currently not using it in CI. # # FIXME(thaJeztah): re-enable this stage when https://github.com/moby/moby/issues/38963 is resolved (see https://github.com/moby/moby/pull/38984) -# COPY --from=criu /build/ /usr/local/bin/ -COPY --from=gotestsum /build/ /usr/local/bin/ -COPY --from=golangci_lint /build/ /usr/local/bin/ -COPY --from=shfmt /build/ /usr/local/bin/ -COPY --from=runc /build/ /usr/local/bin/ -COPY --from=containerd /build/ /usr/local/bin/ -COPY --from=rootlesskit /build/ /usr/local/bin/ -COPY --from=vpnkit / /usr/local/bin/ -COPY --from=containerutil /build/ /usr/local/bin/ -COPY --from=crun /build/ /usr/local/bin/ -COPY hack/dockerfile/etc/docker/ /etc/docker/ +# COPY --link --from=criu /build/ /usr/local/bin/ +COPY --link --from=gotestsum /build/ /usr/local/bin/ +COPY --link --from=golangci_lint /build/ /usr/local/bin/ +COPY --link --from=shfmt /build/ /usr/local/bin/ +COPY --link --from=runc /build/ /usr/local/bin/ +COPY --link --from=containerd /build/ /usr/local/bin/ +COPY --link --from=rootlesskit /build/ /usr/local/bin/ +COPY --link --from=vpnkit / /usr/local/bin/ +COPY --link --from=containerutil /build/ /usr/local/bin/ +COPY --link --from=crun /build/ /usr/local/bin/ +COPY --link hack/dockerfile/etc/docker/ /etc/docker/ ENV PATH=/usr/local/cli:$PATH ENV CONTAINERD_ADDRESS=/run/docker/containerd/containerd.sock ENV CONTAINERD_NAMESPACE=moby @@ -620,13 +620,13 @@ COPY --from=build /build/ / # usage: # > docker buildx bake all FROM scratch AS all -COPY --from=tini /build/ / -COPY --from=runc /build/ / -COPY --from=containerd /build/ / -COPY --from=rootlesskit /build/ / -COPY --from=containerutil /build/ / -COPY --from=vpnkit / / -COPY --from=build /build / +COPY --link --from=tini /build/ / +COPY --link --from=runc /build/ / +COPY --link --from=containerd /build/ / +COPY --link --from=rootlesskit /build/ / +COPY --link --from=containerutil /build/ / +COPY --link --from=vpnkit / / +COPY --link --from=build /build / # smoke tests # usage: From 1fc19772e0ac7da45c17e5044fa1ed88010fed57 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Tue, 30 May 2023 10:20:41 +0200 Subject: [PATCH 055/293] Make sure the image is unpacked for the current snapshotter Switching snapshotter implementations would result in an error when preparing a snapshot, check that the image is indeed unpacked for the current snapshot before trying to prepare a snapshot. Signed-off-by: Djordje Lukic (cherry picked from commit ed32f5e241264a4dc7555f3fe4b301840133ade0) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/image_snapshot.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/daemon/containerd/image_snapshot.go b/daemon/containerd/image_snapshot.go index 35c0816c473c0..a2152505c15c7 100644 --- a/daemon/containerd/image_snapshot.go +++ b/daemon/containerd/image_snapshot.go @@ -3,6 +3,7 @@ package containerd import ( "context" + "github.com/containerd/containerd" containerdimages "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" @@ -12,7 +13,7 @@ import ( // PrepareSnapshot prepares a snapshot from a parent image for a container func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *ocispec.Platform) error { - desc, err := i.resolveDescriptor(ctx, parentImage) + img, err := i.resolveImage(ctx, parentImage) if err != nil { return err } @@ -24,7 +25,19 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentIma matcher = platforms.Only(*platform) } - desc, err = containerdimages.Config(ctx, cs, desc, matcher) + platformImg := containerd.NewImageWithPlatform(i.client, img, matcher) + unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter) + if err != nil { + return err + } + + if !unpacked { + if err := platformImg.Unpack(ctx, i.snapshotter); err != nil { + return err + } + } + + desc, err := containerdimages.Config(ctx, cs, img.Target, matcher) if err != nil { return err } From a27b0381a6a84f53e2c3e5b126cfea8a7eb799f9 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 1 Jun 2023 14:05:20 +0200 Subject: [PATCH 056/293] dockerversion: add a basic unit-test Signed-off-by: Sebastiaan van Stijn (cherry picked from commit eb9a5392bcbcf140bfd29c9ec2ba29e7100a27a4) Signed-off-by: Sebastiaan van Stijn --- dockerversion/useragent_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 dockerversion/useragent_test.go diff --git a/dockerversion/useragent_test.go b/dockerversion/useragent_test.go new file mode 100644 index 0000000000000..86d5004fb7567 --- /dev/null +++ b/dockerversion/useragent_test.go @@ -0,0 +1,24 @@ +package dockerversion + +import ( + "context" + "testing" + + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +func TestDockerUserAgent(t *testing.T) { + t.Run("daemon user-agent", func(t *testing.T) { + ua := DockerUserAgent(context.TODO()) + expected := getDaemonUserAgent() + assert.Check(t, is.Equal(ua, expected)) + }) + + t.Run("daemon user-agent with upstream", func(t *testing.T) { + ctx := context.WithValue(context.TODO(), UAStringKey{}, "Magic-Client/1.2.3 (linux)") + ua := DockerUserAgent(ctx) + expected := getDaemonUserAgent() + ` UpstreamClient(Magic-Client/1.2.3 \(linux\))` + assert.Check(t, is.Equal(ua, expected)) + }) +} From 1d45ea52f42a5521d17beb77576b82c3d345d6ef Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 1 Jun 2023 13:38:45 +0200 Subject: [PATCH 057/293] dockerversion: simplify escapeStr() Use a const for the characters to escape, instead of implementing this as a generic escaping function. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ff40d2d787d2fb8c8d83612cb02aa32d70e6a241) Signed-off-by: Sebastiaan van Stijn --- dockerversion/useragent.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dockerversion/useragent.go b/dockerversion/useragent.go index 7fc3b69a75864..bd65800e99c73 100644 --- a/dockerversion/useragent.go +++ b/dockerversion/useragent.go @@ -69,8 +69,10 @@ func getUserAgentFromContext(ctx context.Context) string { return upstreamUA } +const charsToEscape = `();\` + // escapeStr returns s with every rune in charsToEscape escaped by a backslash -func escapeStr(s string, charsToEscape string) string { +func escapeStr(s string) string { var ret string for _, currRune := range s { appended := false @@ -93,7 +95,5 @@ func escapeStr(s string, charsToEscape string) string { // // $dockerUA UpstreamClient($upstreamUA) func insertUpstreamUserAgent(upstreamUA string, dockerUA string) string { - charsToEscape := `();\` - upstreamUAEscaped := escapeStr(upstreamUA, charsToEscape) - return fmt.Sprintf("%s UpstreamClient(%s)", dockerUA, upstreamUAEscaped) + return fmt.Sprintf("%s UpstreamClient(%s)", dockerUA, escapeStr(upstreamUA)) } From ed376a603fd40cf24935638f9ee5c65eb7e6900c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 1 Jun 2023 13:50:30 +0200 Subject: [PATCH 058/293] dockerversion: remove insertUpstreamUserAgent() It was not really "inserting" anything, just formatting and appending. Simplify this by changing this in to a `getUpstreamUserAgent()` function which returns the upstream User-Agent (if any) into a `UpstreamClient()`. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 9a1f2e6d7cd68d872bdac2cf27cbaad37c9fcd58) Signed-off-by: Sebastiaan van Stijn --- dockerversion/useragent.go | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/dockerversion/useragent.go b/dockerversion/useragent.go index bd65800e99c73..232a28acc9eae 100644 --- a/dockerversion/useragent.go +++ b/dockerversion/useragent.go @@ -18,11 +18,11 @@ type UAStringKey struct{} // // [docker client's UA] UpstreamClient([upstream client's UA]) func DockerUserAgent(ctx context.Context) string { - daemonUA := getDaemonUserAgent() - if upstreamUA := getUserAgentFromContext(ctx); len(upstreamUA) > 0 { - return insertUpstreamUserAgent(upstreamUA, daemonUA) + ua := getDaemonUserAgent() + if upstreamUA := getUpstreamUserAgent(ctx); upstreamUA != "" { + ua += " " + upstreamUA } - return daemonUA + return ua } var ( @@ -57,16 +57,23 @@ func getDaemonUserAgent() string { return daemonUA } -// getUserAgentFromContext returns the previously saved user-agent context stored in ctx, if one exists -func getUserAgentFromContext(ctx context.Context) string { +// getUpstreamUserAgent returns the previously saved user-agent context stored +// in ctx, if one exists, and formats it as: +// +// UpstreamClient() +// +// It returns an empty string if no user-agent is present in the context. +func getUpstreamUserAgent(ctx context.Context) string { var upstreamUA string if ctx != nil { - var ki interface{} = ctx.Value(UAStringKey{}) - if ki != nil { + if ki := ctx.Value(UAStringKey{}); ki != nil { upstreamUA = ctx.Value(UAStringKey{}).(string) } } - return upstreamUA + if upstreamUA == "" { + return "" + } + return fmt.Sprintf("UpstreamClient(%s)", escapeStr(upstreamUA)) } const charsToEscape = `();\` @@ -89,11 +96,3 @@ func escapeStr(s string) string { } return ret } - -// insertUpstreamUserAgent adds the upstream client useragent to create a user-agent -// string of the form: -// -// $dockerUA UpstreamClient($upstreamUA) -func insertUpstreamUserAgent(upstreamUA string, dockerUA string) string { - return fmt.Sprintf("%s UpstreamClient(%s)", dockerUA, escapeStr(upstreamUA)) -} From 8018ee46894814a27c438d08b3ecfa2326412479 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 1 Jun 2023 14:09:50 +0200 Subject: [PATCH 059/293] dockerversion: DockerUserAgent(): allow custom versions to be passed Allow additional metadata to be passed as part of the generated User-Agent. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit a6da1480b5197ad0d2dc7194b117d8dfe615df44) Signed-off-by: Sebastiaan van Stijn --- dockerversion/useragent.go | 4 ++-- dockerversion/useragent_test.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dockerversion/useragent.go b/dockerversion/useragent.go index 232a28acc9eae..7013a4543df3b 100644 --- a/dockerversion/useragent.go +++ b/dockerversion/useragent.go @@ -17,8 +17,8 @@ type UAStringKey struct{} // In accordance with RFC 7231 (5.5.3) is of the form: // // [docker client's UA] UpstreamClient([upstream client's UA]) -func DockerUserAgent(ctx context.Context) string { - ua := getDaemonUserAgent() +func DockerUserAgent(ctx context.Context, extraVersions ...useragent.VersionInfo) string { + ua := useragent.AppendVersions(getDaemonUserAgent(), extraVersions...) if upstreamUA := getUpstreamUserAgent(ctx); upstreamUA != "" { ua += " " + upstreamUA } diff --git a/dockerversion/useragent_test.go b/dockerversion/useragent_test.go index 86d5004fb7567..b9fe3d2dfac3a 100644 --- a/dockerversion/useragent_test.go +++ b/dockerversion/useragent_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/docker/docker/pkg/useragent" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" ) @@ -15,10 +16,23 @@ func TestDockerUserAgent(t *testing.T) { assert.Check(t, is.Equal(ua, expected)) }) + t.Run("daemon user-agent custom metadata", func(t *testing.T) { + ua := DockerUserAgent(context.TODO(), useragent.VersionInfo{Name: "hello", Version: "world"}, useragent.VersionInfo{Name: "foo", Version: "bar"}) + expected := getDaemonUserAgent() + ` hello/world foo/bar` + assert.Check(t, is.Equal(ua, expected)) + }) + t.Run("daemon user-agent with upstream", func(t *testing.T) { ctx := context.WithValue(context.TODO(), UAStringKey{}, "Magic-Client/1.2.3 (linux)") ua := DockerUserAgent(ctx) expected := getDaemonUserAgent() + ` UpstreamClient(Magic-Client/1.2.3 \(linux\))` assert.Check(t, is.Equal(ua, expected)) }) + + t.Run("daemon user-agent with upstream and custom metadata", func(t *testing.T) { + ctx := context.WithValue(context.TODO(), UAStringKey{}, "Magic-Client/1.2.3 (linux)") + ua := DockerUserAgent(ctx, useragent.VersionInfo{Name: "hello", Version: "world"}, useragent.VersionInfo{Name: "foo", Version: "bar"}) + expected := getDaemonUserAgent() + ` hello/world foo/bar UpstreamClient(Magic-Client/1.2.3 \(linux\))` + assert.Check(t, is.Equal(ua, expected)) + }) } From 75afe3201b3dc08a3739e2a76fe315c03f33199b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 1 Jun 2023 14:16:22 +0200 Subject: [PATCH 060/293] containerd: add c8d version and storage-driver to User-Agent With this patch, the user-agent has information about the containerd-client version and the storage-driver that's used when using the containerd-integration; time="2023-06-01T11:27:07.959822887Z" level=info msg="listening on [::]:5000" go.version=go1.19.9 instance.id=53590f34-096a-4fd1-9c58-d3b8eb7e5092 service=registry version=2.8.2 ... 172.18.0.1 - - [01/Jun/2023:11:30:12 +0000] "HEAD /v2/multifoo/blobs/sha256:c7ec7661263e5e597156f2281d97b160b91af56fa1fd2cc045061c7adac4babd HTTP/1.1" 404 157 "" "docker/dev go/go1.20.4 git-commit/8d67d0c1a8 kernel/5.15.49-linuxkit-pr os/linux arch/arm64 containerd-client/1.6.21+unknown storage-driver/overlayfs UpstreamClient(Docker-Client/24.0.2 \\(linux\\))" Signed-off-by: Sebastiaan van Stijn (cherry picked from commit d099e47e00ee1a34446d74561279a4fba5417ee2) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/resolver.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/daemon/containerd/resolver.go b/daemon/containerd/resolver.go index 97bc9780df31b..db44b03dff0a6 100644 --- a/daemon/containerd/resolver.go +++ b/daemon/containerd/resolver.go @@ -8,8 +8,10 @@ import ( "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" + "github.com/containerd/containerd/version" registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/dockerversion" + "github.com/docker/docker/pkg/useragent" "github.com/docker/docker/registry" "github.com/sirupsen/logrus" ) @@ -20,7 +22,7 @@ func (i *ImageService) newResolverFromAuthConfig(ctx context.Context, authConfig hosts := hostsWrapper(hostsFn, authConfig, i.registryService) headers := http.Header{} - headers.Set("User-Agent", dockerversion.DockerUserAgent(ctx)) + headers.Set("User-Agent", dockerversion.DockerUserAgent(ctx, useragent.VersionInfo{Name: "containerd-client", Version: version.Version}, useragent.VersionInfo{Name: "storage-driver", Version: i.snapshotter})) return docker.NewResolver(docker.ResolverOptions{ Hosts: hosts, From e1c7956764a8bdb9a9513b37ed3b75d2eb24fe83 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 1 Jun 2023 22:14:38 +0200 Subject: [PATCH 061/293] Dockerfile: use COPY --link for source code as well I missed the most important COPY in 637ca59375cdc26486e9451eea1aa6a9aac6154e Copying the source code into the dev-container does not depend on the parent layers, so can use the --link option as well. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ff2342154b9c34925e0ddb7988b6925346d9efcb) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 56350d6d0feb6..a9f9725e61b8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -646,4 +646,4 @@ EOT # > make shell # > SYSTEMD=true make shell FROM dev-base AS dev -COPY . . +COPY --link . . From 0139309fef8de56d19977b05b5693b30b50d08d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 29 May 2023 11:20:06 +0200 Subject: [PATCH 062/293] c8d: Add walkImageManifests and ImageManifest wrapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default implementation of the containerd.Image interface provided by the containerd operates on the parent index/manifest list of the image and the platform matcher. This isn't convenient when a specific manifest is already known and it's redundant to search the whole index for a manifest that matches the given platform matcher. It can also result in a different manifest picked up than expected when multiple manifests with the same platform are present. This introduces a walkImageManifests which walks the provided image and calls a handler with a ImageManifest, which is a simple wrapper that implements containerd.Image interfaces and performs all containerd.Image operations against a platform specific manifest instead of the root manifest list/index. Signed-off-by: Paweł Gronowski (cherry picked from commit fabc1d5bef1dbe4a8779fef04cf3897a0c793372) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_manifest.go | 146 ++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 daemon/containerd/image_manifest.go diff --git a/daemon/containerd/image_manifest.go b/daemon/containerd/image_manifest.go new file mode 100644 index 0000000000000..7ec7cf311785b --- /dev/null +++ b/daemon/containerd/image_manifest.go @@ -0,0 +1,146 @@ +package containerd + +import ( + "context" + "encoding/json" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + containerdimages "github.com/containerd/containerd/images" + cplatforms "github.com/containerd/containerd/platforms" + "github.com/docker/docker/errdefs" + "github.com/moby/buildkit/util/attestation" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// walkImageManifests calls the handler for each locally present manifest in +// the image. The image implements the containerd.Image interface, but all +// operations act on the specific manifest instead of the index. +func (i *ImageService) walkImageManifests(ctx context.Context, img containerdimages.Image, handler func(img *ImageManifest) error) error { + desc := img.Target + + handleManifest := func(ctx context.Context, d ocispec.Descriptor) error { + platformImg, err := i.NewImageManifest(ctx, img, d) + if err != nil { + return err + } + return handler(platformImg) + } + + if containerdimages.IsIndexType(desc.MediaType) { + store := i.client.ContentStore() + return containerdimages.Walk(ctx, presentChildrenHandler(store, containerdimages.HandlerFunc( + func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if containerdimages.IsManifestType(desc.MediaType) { + return nil, handleManifest(ctx, desc) + } + return nil, nil + })), desc) + } + + return handleManifest(ctx, desc) +} + +type ImageManifest struct { + containerd.Image + + // Parent of the manifest (index/manifest list) + RealTarget ocispec.Descriptor + + manifest *ocispec.Manifest +} + +func (i *ImageService) NewImageManifest(ctx context.Context, img containerdimages.Image, manifestDesc ocispec.Descriptor) (*ImageManifest, error) { + if !containerdimages.IsManifestType(manifestDesc.MediaType) { + return nil, errdefs.InvalidParameter(errors.New("descriptor isn't a manifest")) + } + + parent := img.Target + img.Target = manifestDesc + + c8dImg := containerd.NewImageWithPlatform(i.client, img, cplatforms.All) + return &ImageManifest{ + Image: c8dImg, + RealTarget: parent, + }, nil +} + +func (im *ImageManifest) Metadata() containerdimages.Image { + md := im.Image.Metadata() + md.Target = im.RealTarget + return md +} + +// IsPseudoImage returns false if the manifest has no layers or any of its layers is a known image layer. +// Some manifests use the image media type for compatibility, even if they are not a real image. +func (im *ImageManifest) IsPseudoImage(ctx context.Context) (bool, error) { + desc := im.Target() + + // Quick check for buildkit attestation manifests + // https://github.com/moby/buildkit/blob/v0.11.4/docs/attestations/attestation-storage.md + // This would have also been caught by the layer check below, but it requires + // an additional content read and deserialization of Manifest. + if _, has := desc.Annotations[attestation.DockerAnnotationReferenceType]; has { + return true, nil + } + + mfst, err := im.Manifest(ctx) + if err != nil { + return true, err + } + if len(mfst.Layers) == 0 { + return false, nil + } + for _, l := range mfst.Layers { + if images.IsLayerType(l.MediaType) { + return false, nil + } + } + return true, nil +} + +func (im *ImageManifest) Manifest(ctx context.Context) (ocispec.Manifest, error) { + if im.manifest != nil { + return *im.manifest, nil + } + + mfst, err := readManifest(ctx, im.ContentStore(), im.Target()) + if err != nil { + return ocispec.Manifest{}, err + } + + im.manifest = &mfst + return mfst, nil +} + +func (im *ImageManifest) CheckContentAvailable(ctx context.Context) (bool, error) { + // The target is already a platform-specific manifest, so no need to match platform. + pm := cplatforms.All + + available, _, _, missing, err := containerdimages.Check(ctx, im.ContentStore(), im.Target(), pm) + if err != nil { + return false, err + } + + if !available || len(missing) > 0 { + return false, nil + } + + return true, nil +} + +func readManifest(ctx context.Context, store content.Provider, desc ocispec.Descriptor) (ocispec.Manifest, error) { + p, err := content.ReadBlob(ctx, store, desc) + if err != nil { + return ocispec.Manifest{}, err + } + + var mfst ocispec.Manifest + if err := json.Unmarshal(p, &mfst); err != nil { + return ocispec.Manifest{}, err + } + + return mfst, nil +} From cbf0779bfc215e4ef7f6c78e6da21374891b4950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 29 May 2023 14:14:52 +0200 Subject: [PATCH 063/293] c8d/list: Use walkImageManifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 5210f48bfc93373d93160b25509189654d338a46) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_list.go | 118 ++++++++------------------------ 1 file changed, 27 insertions(+), 91 deletions(-) diff --git a/daemon/containerd/image_list.go b/daemon/containerd/image_list.go index a1da3eb2c5625..e8afa43945a46 100644 --- a/daemon/containerd/image_list.go +++ b/daemon/containerd/image_list.go @@ -6,17 +6,14 @@ import ( "strings" "time" - "github.com/containerd/containerd" "github.com/containerd/containerd/content" cerrdefs "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/labels" - "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" timetypes "github.com/docker/docker/api/types/time" - "github.com/moby/buildkit/util/attestation" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -87,72 +84,41 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions) continue } - err := images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - if images.IsIndexType(desc.MediaType) { - return images.Children(ctx, contentStore, desc) + err := i.walkImageManifests(ctx, img, func(img *ImageManifest) error { + if isPseudo, err := img.IsPseudoImage(ctx); isPseudo || err != nil { + return err } - if images.IsManifestType(desc.MediaType) { - // Ignore buildkit attestation manifests - // https://github.com/moby/buildkit/blob/v0.11.4/docs/attestations/attestation-storage.md - // This would have also been caught by the isImageManifest call below, but it requires - // an additional content read and deserialization of Manifest. - if _, has := desc.Annotations[attestation.DockerAnnotationReferenceType]; has { - return nil, nil - } - - mfst, err := images.Manifest(ctx, contentStore, desc, platforms.All) - if err != nil { - if cerrdefs.IsNotFound(err) { - return nil, nil - } - return nil, err - } - - if !isImageManifest(mfst) { - return nil, nil - } - - platform, err := getManifestPlatform(ctx, contentStore, desc, mfst.Config) - if err != nil { - if cerrdefs.IsNotFound(err) { - return nil, nil - } - return nil, err - } + available, err := img.CheckContentAvailable(ctx) + if err != nil { + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "manifest": img.Target(), + "image": img.Name(), + }).Warn("checking availability of platform specific manifest failed") + return nil + } - pm := platforms.OnlyStrict(platform) - available, _, _, missing, err := images.Check(ctx, contentStore, img.Target, pm) - if err != nil { - logrus.WithFields(logrus.Fields{ - logrus.ErrorKey: err, - "platform": platform, - "image": img.Target, - }).Warn("checking availability of platform content failed") - return nil, nil - } - if !available || len(missing) > 0 { - return nil, nil - } + if !available { + return nil + } - c8dImage := containerd.NewImageWithPlatform(i.client, img, pm) - image, chainIDs, err := i.singlePlatformImage(ctx, contentStore, c8dImage) - if err != nil { - return nil, err - } + image, chainIDs, err := i.singlePlatformImage(ctx, contentStore, img) + if err != nil { + return err + } - summaries = append(summaries, image) + summaries = append(summaries, image) - if opts.SharedSize { - root = append(root, &chainIDs) - for _, id := range chainIDs { - layers[id] = layers[id] + 1 - } + if opts.SharedSize { + root = append(root, &chainIDs) + for _, id := range chainIDs { + layers[id] = layers[id] + 1 } } - return nil, nil - }), img.Target) + return nil + }) if err != nil { return nil, err @@ -173,7 +139,7 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions) return summaries, nil } -func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, image containerd.Image) (*types.ImageSummary, []digest.Digest, error) { +func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, image *ImageManifest) (*types.ImageSummary, []digest.Digest, error) { diffIDs, err := image.RootFS(ctx) if err != nil { return nil, nil, err @@ -509,36 +475,6 @@ func computeSharedSize(chainIDs []digest.Digest, layers map[digest.Digest]int, s return sharedSize, nil } -// getManifestPlatform returns a platform specified by the manifest descriptor -// or reads it from its config. -func getManifestPlatform(ctx context.Context, store content.Provider, manifestDesc, configDesc ocispec.Descriptor) (ocispec.Platform, error) { - var platform ocispec.Platform - if manifestDesc.Platform != nil { - platform = *manifestDesc.Platform - } else { - // Config is technically a v1.Image, but it has the same member as v1.Platform - // which makes the v1.Platform a subset of Image so we can unmarshal directly. - if err := readConfig(ctx, store, configDesc, &platform); err != nil { - return platform, err - } - } - return platforms.Normalize(platform), nil -} - -// isImageManifest returns true if the manifest has no layers or any of its layers is a known image layer. -// Some manifests use the image media type for compatibility, even if they are not a real image. -func isImageManifest(mfst ocispec.Manifest) bool { - if len(mfst.Layers) == 0 { - return true - } - for _, l := range mfst.Layers { - if images.IsLayerType(l.MediaType) { - return true - } - } - return false -} - // readConfig reads content pointed by the descriptor and unmarshals it into a specified output. func readConfig(ctx context.Context, store content.Provider, desc ocispec.Descriptor, out interface{}) error { data, err := content.ReadBlob(ctx, store, desc) From 0b9d68f59d245f3da93a6ab733e7a0b2e2e1b28e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 29 May 2023 14:15:11 +0200 Subject: [PATCH 064/293] c8d/load: Use walkImageManifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit b08bff8ba3c3c1a8152e6f9ae02b8255a9c637a0) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_exporter.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/daemon/containerd/image_exporter.go b/daemon/containerd/image_exporter.go index efb9e4e5c4f6d..fd42cce828e61 100644 --- a/daemon/containerd/image_exporter.go +++ b/daemon/containerd/image_exporter.go @@ -18,6 +18,7 @@ import ( "github.com/docker/docker/pkg/streamformatter" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -125,16 +126,9 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt return errdefs.System(err) } - store := i.client.ContentStore() progress := streamformatter.NewStdoutWriter(outStream) for _, img := range imgs { - allPlatforms, err := containerdimages.Platforms(ctx, store, img.Target) - if err != nil { - logrus.WithError(err).WithField("image", img.Name).Debug("failed to get image platforms") - return errdefs.Unknown(err) - } - name := img.Name loadedMsg := "Loaded image" @@ -145,17 +139,16 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt name = reference.FamiliarName(reference.TagNameOnly(named)) } - for _, platform := range allPlatforms { + err = i.walkImageManifests(ctx, img, func(platformImg *ImageManifest) error { logger := logrus.WithFields(logrus.Fields{ - "platform": platform, "image": name, + "manifest": platformImg.Target().Digest, }) - platformImg := containerd.NewImageWithPlatform(i.client, img, cplatforms.OnlyStrict(platform)) unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter) if err != nil { - logger.WithError(err).Debug("failed to check if image is unpacked") - continue + logger.WithError(err).Warn("failed to check if image is unpacked") + return nil } if !unpacked { @@ -166,6 +159,10 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt } } logger.WithField("alreadyUnpacked", unpacked).WithError(err).Debug("unpack") + return nil + }) + if err != nil { + return errors.Wrap(err, "failed to unpack loaded image") } fmt.Fprintf(progress, "%s: %s\n", loadedMsg, name) From 087cf6f238506f2d08d73f141cc9fb4cb9fd5826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 29 May 2023 14:17:19 +0200 Subject: [PATCH 065/293] c8d/load: Don't unpack pseudo images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't unpack image manifests which are not a real images that can't be unpacked. Signed-off-by: Paweł Gronowski (cherry picked from commit 4d3238dc0b77d6df2b308d175cf1dec9d364dddf) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_exporter.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/daemon/containerd/image_exporter.go b/daemon/containerd/image_exporter.go index fd42cce828e61..2eec15c7b8005 100644 --- a/daemon/containerd/image_exporter.go +++ b/daemon/containerd/image_exporter.go @@ -145,6 +145,15 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt "manifest": platformImg.Target().Digest, }) + if isPseudo, err := platformImg.IsPseudoImage(ctx); isPseudo || err != nil { + if err != nil { + logger.WithError(err).Warn("failed to read manifest") + } else { + logger.Debug("don't unpack non-image manifest") + } + return nil + } + unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter) if err != nil { logger.WithError(err).Warn("failed to check if image is unpacked") From 961fe2740821abf73dd3328e327bcbffd92bd53a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 2 Jun 2023 10:23:03 +0200 Subject: [PATCH 066/293] c8d/handlers: Handle error in walkPresentChildren MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 429580673696e47e108369b2d4c3343cb083b0a5) Signed-off-by: Paweł Gronowski --- daemon/containerd/handlers.go | 9 +++------ daemon/containerd/image_delete.go | 3 ++- daemon/containerd/image_manifest.go | 25 +++++++++++++++---------- daemon/containerd/image_prune.go | 6 ++++-- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/daemon/containerd/handlers.go b/daemon/containerd/handlers.go index b77b7b738b429..0fd571a27950e 100644 --- a/daemon/containerd/handlers.go +++ b/daemon/containerd/handlers.go @@ -9,16 +9,13 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// walkPresentChildren is a simple wrapper for containerdimages.Walk with -// presentChildrenHandler wrapping a simple handler that only operates on -// walked Descriptor and doesn't return any errror. +// walkPresentChildren is a simple wrapper for containerdimages.Walk with presentChildrenHandler. // This is only a convenient helper to reduce boilerplate. -func (i *ImageService) walkPresentChildren(ctx context.Context, target ocispec.Descriptor, f func(context.Context, ocispec.Descriptor)) error { +func (i *ImageService) walkPresentChildren(ctx context.Context, target ocispec.Descriptor, f func(context.Context, ocispec.Descriptor) error) error { store := i.client.ContentStore() return containerdimages.Walk(ctx, presentChildrenHandler(store, containerdimages.HandlerFunc( func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - f(ctx, desc) - return nil, nil + return nil, f(ctx, desc) })), target) } diff --git a/daemon/containerd/image_delete.go b/daemon/containerd/image_delete.go index 3d55dba3948ca..7566cdbd9b80a 100644 --- a/daemon/containerd/image_delete.go +++ b/daemon/containerd/image_delete.go @@ -124,10 +124,11 @@ func (i *ImageService) deleteAll(ctx context.Context, img images.Image, force, p // Workaround for: https://github.com/moby/buildkit/issues/3797 possiblyDeletedConfigs := map[digest.Digest]struct{}{} - err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) { + err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) error { if images.IsConfigType(d.MediaType) { possiblyDeletedConfigs[d.Digest] = struct{}{} } + return nil }) if err != nil { return nil, err diff --git a/daemon/containerd/image_manifest.go b/daemon/containerd/image_manifest.go index 7ec7cf311785b..d5e42113f6fde 100644 --- a/daemon/containerd/image_manifest.go +++ b/daemon/containerd/image_manifest.go @@ -15,6 +15,11 @@ import ( "github.com/pkg/errors" ) +var ( + errNotManifestOrIndex = errdefs.InvalidParameter(errors.New("descriptor is neither a manifest or index")) + errNotManifest = errdefs.InvalidParameter(errors.New("descriptor isn't a manifest")) +) + // walkImageManifests calls the handler for each locally present manifest in // the image. The image implements the containerd.Image interface, but all // operations act on the specific manifest instead of the index. @@ -24,23 +29,23 @@ func (i *ImageService) walkImageManifests(ctx context.Context, img containerdima handleManifest := func(ctx context.Context, d ocispec.Descriptor) error { platformImg, err := i.NewImageManifest(ctx, img, d) if err != nil { + if err == errNotManifest { + return nil + } return err } return handler(platformImg) } + if containerdimages.IsManifestType(desc.MediaType) { + return handleManifest(ctx, desc) + } + if containerdimages.IsIndexType(desc.MediaType) { - store := i.client.ContentStore() - return containerdimages.Walk(ctx, presentChildrenHandler(store, containerdimages.HandlerFunc( - func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - if containerdimages.IsManifestType(desc.MediaType) { - return nil, handleManifest(ctx, desc) - } - return nil, nil - })), desc) + return i.walkPresentChildren(ctx, desc, handleManifest) } - return handleManifest(ctx, desc) + return errNotManifestOrIndex } type ImageManifest struct { @@ -54,7 +59,7 @@ type ImageManifest struct { func (i *ImageService) NewImageManifest(ctx context.Context, img containerdimages.Image, manifestDesc ocispec.Descriptor) (*ImageManifest, error) { if !containerdimages.IsManifestType(manifestDesc.MediaType) { - return nil, errdefs.InvalidParameter(errors.New("descriptor isn't a manifest")) + return nil, errNotManifest } parent := img.Target diff --git a/daemon/containerd/image_prune.go b/daemon/containerd/image_prune.go index 6da0d2c013135..d4c2052a13c3f 100644 --- a/daemon/containerd/image_prune.go +++ b/daemon/containerd/image_prune.go @@ -125,11 +125,12 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu blobs := []ocispec.Descriptor{} - err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) { + err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) error { blobs = append(blobs, desc) if containerdimages.IsConfigType(desc.MediaType) { possiblyDeletedConfigs[desc.Digest] = struct{}{} } + return nil }) if err != nil { errs = multierror.Append(errs, err) @@ -186,10 +187,11 @@ func (i *ImageService) unleaseSnapshotsFromDeletedConfigs(ctx context.Context, p var errs error for _, img := range all { - err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) { + err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) error { if containerdimages.IsConfigType(desc.MediaType) { delete(possiblyDeletedConfigs, desc.Digest) } + return nil }) if err != nil { errs = multierror.Append(errs, err) From 647ba0322473b1f0a8d7a13875051d547d112bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 1 Jun 2023 00:35:59 +0200 Subject: [PATCH 067/293] builder-next: Set moby exporter as default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit d63569c73d467af1030b166e1a0e730e49bfbf4e) Signed-off-by: Paweł Gronowski --- builder/builder-next/builder.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/builder/builder-next/builder.go b/builder/builder-next/builder.go index f6a89633e6570..f2a4ef0c9db0f 100644 --- a/builder/builder-next/builder.go +++ b/builder/builder-next/builder.go @@ -355,11 +355,7 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. exporterName := "" exporterAttrs := map[string]string{} if len(opt.Options.Outputs) == 0 { - if b.useSnapshotter { - exporterName = client.ExporterImage - } else { - exporterName = exporter.Moby - } + exporterName = exporter.Moby } else { // cacheonly is a special type for triggering skipping all exporters if opt.Options.Outputs[0].Type != "cacheonly" { From c4198e6053b1d4840f9c6e6dd152314548539356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 19 Apr 2023 14:48:47 +0200 Subject: [PATCH 068/293] Dockerfile: Use separate cli for shell and integration-cli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use separate cli for integration-cli to allow use newer CLI for interactive dev shell usage. Both versions can be overriden with DOCKERCLI_VERSION or DOCKERCLI_INTEGRATION_VERSION. Binary is downloaded from download.docker.com if it's available, otherwise it's built from the source. For backwards compatibility DOCKER_CLI_PATH overrides BOTH clis. Signed-off-by: Paweł Gronowski (cherry picked from commit 17c99f716458575cdfb6311067720e3b6e5f8bc0) Signed-off-by: Paweł Gronowski --- Dockerfile | 50 ++++++++++++++--------------- Makefile | 8 +++++ hack/dockerfile/cli.sh | 29 +++++++++++++++++ hack/make/.integration-daemon-start | 6 ++-- integration-cli/check_test.go | 14 ++++++++ 5 files changed, 79 insertions(+), 28 deletions(-) create mode 100755 hack/dockerfile/cli.sh diff --git a/Dockerfile b/Dockerfile index a9f9725e61b8d..4b3171c0ef439 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,12 @@ ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" ARG XX_VERSION=1.2.1 ARG VPNKIT_VERSION=0.5.0 -ARG DOCKERCLI_VERSION=v17.06.2-ce + +ARG DOCKERCLI_REPOSITORY="https://github.com/docker/cli.git" +ARG DOCKERCLI_VERSION=v24.0.2 +# cli version used for integration-cli tests +ARG DOCKERCLI_INTEGRATION_REPOSITORY="https://github.com/docker/cli.git" +ARG DOCKERCLI_INTEGRATION_VERSION=v17.06.2-ce ARG SYSTEMD="false" ARG DEBIAN_FRONTEND=noninteractive @@ -243,34 +248,25 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ GOBIN=/build/ GO111MODULE=on go install "mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION}" \ && /build/shfmt --version -# dockercli -FROM base AS dockercli-src -WORKDIR /tmp/dockercli -RUN git init . && git remote add origin "https://github.com/docker/cli.git" -ARG DOCKERCLI_VERSION -RUN git fetch -q --depth 1 origin "${DOCKERCLI_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD -RUN [ -d ./components/cli ] && mv ./components/cli /usr/src/dockercli || mv /tmp/dockercli /usr/src/dockercli -WORKDIR /usr/src/dockercli - FROM base AS dockercli WORKDIR /go/src/github.com/docker/cli +COPY hack/dockerfile/cli.sh /download-or-build-cli.sh +ARG DOCKERCLI_REPOSITORY ARG DOCKERCLI_VERSION -ARG DOCKERCLI_CHANNEL=stable ARG TARGETPLATFORM -RUN xx-apt-get install -y --no-install-recommends gcc libc6-dev -RUN --mount=from=dockercli-src,src=/usr/src/dockercli,rw \ - --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM </dev/null 2>&1; then - mkdir /build - curl -Ls "${DOWNLOAD_URL}" | tar -xz docker/docker - mv docker/docker /build/docker - else - CGO_ENABLED=0 xx-go build -o /build/docker ./cmd/docker - fi - xx-verify /build/docker -EOT +RUN --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,target=./.git \ + --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM \ + /download-or-build-cli.sh ${DOCKERCLI_VERSION} ${DOCKERCLI_REPOSITORY} /build + +FROM base AS dockercli-integration +WORKDIR /go/src/github.com/docker/cli +COPY hack/dockerfile/cli.sh /download-or-build-cli.sh +ARG DOCKERCLI_INTEGRATION_REPOSITORY +ARG DOCKERCLI_INTEGRATION_VERSION +ARG TARGETPLATFORM +RUN --mount=type=cache,id=dockercli-integration-git-$TARGETPLATFORM,target=./.git \ + --mount=type=cache,target=/root/.cache/go-build,id=dockercli-integration-build-$TARGETPLATFORM \ + /download-or-build-cli.sh ${DOCKERCLI_INTEGRATION_VERSION} ${DOCKERCLI_INTEGRATION_REPOSITORY} /build # runc FROM base AS runc-src @@ -439,7 +435,6 @@ FROM containerutil-windows-${TARGETARCH} AS containerutil-windows FROM containerutil-${TARGETOS} AS containerutil FROM base AS dev-systemd-false -COPY --link --from=dockercli /build/ /usr/local/cli COPY --link --from=frozen-images /build/ /docker-frozen-images COPY --link --from=swagger /build/ /usr/local/bin/ COPY --link --from=delve /build/ /usr/local/bin/ @@ -464,11 +459,14 @@ COPY --link --from=containerutil /build/ /usr/local/bin/ COPY --link --from=crun /build/ /usr/local/bin/ COPY --link hack/dockerfile/etc/docker/ /etc/docker/ ENV PATH=/usr/local/cli:$PATH +ENV TEST_CLIENT_BINARY=/usr/local/cli-integration/docker ENV CONTAINERD_ADDRESS=/run/docker/containerd/containerd.sock ENV CONTAINERD_NAMESPACE=moby WORKDIR /go/src/github.com/docker/docker VOLUME /var/lib/docker VOLUME /home/unprivilegeduser/.local/share/docker +COPY --link --from=dockercli /build/ /usr/local/cli +COPY --link --from=dockercli-integration /build/ /usr/local/cli-integration # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] diff --git a/Makefile b/Makefile index 97ad5915f3037..17aebbd26c6b7 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,10 @@ DOCKER_ENVS := \ -e DOCKER_BUILDKIT \ -e DOCKER_BASH_COMPLETION_PATH \ -e DOCKER_CLI_PATH \ + -e DOCKERCLI_VERSION \ + -e DOCKERCLI_REPOSITORY \ + -e DOCKERCLI_INTEGRATION_VERSION \ + -e DOCKERCLI_INTEGRATION_REPOSITORY \ -e DOCKER_DEBUG \ -e DOCKER_EXPERIMENTAL \ -e DOCKER_GITCOMMIT \ @@ -136,6 +140,10 @@ endif DOCKER_RUN_DOCKER := $(DOCKER_FLAGS) "$(DOCKER_IMAGE)" DOCKER_BUILD_ARGS += --build-arg=GO_VERSION +DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_VERSION +DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_REPOSITORY +DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_INTEGRATION_VERSION +DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_INTEGRATION_REPOSITORY ifdef DOCKER_SYSTEMD DOCKER_BUILD_ARGS += --build-arg=SYSTEMD=true endif diff --git a/hack/dockerfile/cli.sh b/hack/dockerfile/cli.sh new file mode 100755 index 0000000000000..6af829210a1da --- /dev/null +++ b/hack/dockerfile/cli.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e +version="$1" +repository="$2" +outdir="$3" + +DOWNLOAD_URL="https://download.docker.com/linux/static/stable/$(xx-info march)/docker-${version#v}.tgz" + +mkdir "$outdir" +if curl --head --silent --fail "${DOWNLOAD_URL}" 1> /dev/null 2>&1; then + curl -Ls "${DOWNLOAD_URL}" | tar -xz docker/docker + mv docker/docker "${outdir}/docker" +else + git init -q . + git remote remove origin || true + git remote add origin "${repository}" + git fetch -q --depth 1 origin "${version}" +refs/tags/*:refs/tags/* + git checkout -fq "${version}" + if [ -d ./components/cli ]; then + mv ./components/cli/* ./ + CGO_ENABLED=0 xx-go build -o "${outdir}/docker" ./cmd/docker + git reset --hard "${version}" + else + xx-go --wrap && CGO_ENABLED=0 TARGET="${outdir}" ./scripts/build/binary + fi +fi + +xx-verify "${outdir}/docker" diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index 766e09f7fb240..89e38993dbbe1 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -5,9 +5,11 @@ base="$ABS_DEST/.." export PATH="$base/dynbinary-daemon:$base/binary-daemon:$PATH" -export TEST_CLIENT_BINARY=docker - +if [ -z "$TEST_CLIENT_BINARY" ]; then + export TEST_CLIENT_BINARY=docker +fi if [ -n "$DOCKER_CLI_PATH" ]; then + # /usr/local/cli is a bind mount to the base dir of DOCKER_CLI_PATH (if used) export TEST_CLIENT_BINARY=/usr/local/cli/$(basename "$DOCKER_CLI_PATH") fi diff --git a/integration-cli/check_test.go b/integration-cli/check_test.go index db6683709ddcf..dd036f3f0e01b 100644 --- a/integration-cli/check_test.go +++ b/integration-cli/check_test.go @@ -67,9 +67,23 @@ func TestMain(m *testing.M) { } testEnv.Print() + printCliVersion() os.Exit(m.Run()) } +func printCliVersion() { + // Print output of "docker version" + cli.SetTestEnvironment(testEnv) + cmd := cli.Docker(cli.Args("version")) + if cmd.Error != nil { + fmt.Printf("WARNING: Failed to run \"docker version\": %+v\n", cmd.Error) + return + } + + fmt.Println("INFO: Testing with docker cli version:") + fmt.Println(cmd.Stdout()) +} + func ensureTestEnvSetup(t *testing.T) { testEnvOnce.Do(func() { cli.SetTestEnvironment(testEnv) From 1a078977e1e5d02883917c1ac1c7054825bf4314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 1 Feb 2023 14:17:36 +0100 Subject: [PATCH 069/293] Dockerfile/shell: Install buildx cli plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installs the buildx cli plugin in the container shell by default. Previously user had to manually download the buildx binary to use buildkit. Signed-off-by: Paweł Gronowski (cherry picked from commit 49f76a34b53700a4724dd71ed4348f6068c4c70b) Signed-off-by: Paweł Gronowski --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4b3171c0ef439..38bde1c40d0ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ ARG DOCKERCLI_VERSION=v24.0.2 # cli version used for integration-cli tests ARG DOCKERCLI_INTEGRATION_REPOSITORY="https://github.com/docker/cli.git" ARG DOCKERCLI_INTEGRATION_VERSION=v17.06.2-ce +ARG BUILDX_VERSION=0.10.5 ARG SYSTEMD="false" ARG DEBIAN_FRONTEND=noninteractive @@ -433,6 +434,7 @@ FROM binary-dummy AS containerutil-linux FROM containerutil-build AS containerutil-windows-amd64 FROM containerutil-windows-${TARGETARCH} AS containerutil-windows FROM containerutil-${TARGETOS} AS containerutil +FROM docker/buildx-bin:${BUILDX_VERSION} as buildx FROM base AS dev-systemd-false COPY --link --from=frozen-images /build/ /docker-frozen-images @@ -458,6 +460,8 @@ COPY --link --from=vpnkit / /usr/local/bin/ COPY --link --from=containerutil /build/ /usr/local/bin/ COPY --link --from=crun /build/ /usr/local/bin/ COPY --link hack/dockerfile/etc/docker/ /etc/docker/ +COPY --link --from=buildx /buildx /usr/local/libexec/docker/cli-plugins/docker-buildx + ENV PATH=/usr/local/cli:$PATH ENV TEST_CLIENT_BINARY=/usr/local/cli-integration/docker ENV CONTAINERD_ADDRESS=/run/docker/containerd/containerd.sock From e5fbc3f75ae155651a073ace8341d2d0571e36d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 5 Jun 2023 16:31:15 +0200 Subject: [PATCH 070/293] hack/cli.sh: Quiet origin cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't show `error: No such remote: 'origin'` error when building for the first time and the cached git repository doesn't a remote yet. Signed-off-by: Paweł Gronowski (cherry picked from commit 5433b88e2dbd43852a0f1f9625fefcf69a6ea688) Signed-off-by: Paweł Gronowski --- hack/dockerfile/cli.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/dockerfile/cli.sh b/hack/dockerfile/cli.sh index 6af829210a1da..f821994650270 100755 --- a/hack/dockerfile/cli.sh +++ b/hack/dockerfile/cli.sh @@ -13,7 +13,7 @@ if curl --head --silent --fail "${DOWNLOAD_URL}" 1> /dev/null 2>&1; then mv docker/docker "${outdir}/docker" else git init -q . - git remote remove origin || true + git remote remove origin 2> /dev/null || true git remote add origin "${repository}" git fetch -q --depth 1 origin "${version}" +refs/tags/*:refs/tags/* git checkout -fq "${version}" From 61d547fd06cbbe362dd94c22d85d5e44ae635611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 5 Jun 2023 16:32:24 +0200 Subject: [PATCH 071/293] Dockerfile: Move dockercli to base-dev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoids invalidation of dev-systemd-true and dev-base when changing the CLI version/repository. Signed-off-by: Paweł Gronowski (cherry picked from commit 0f9c8e684a3132b0cfe1f3a04fec68e5f8f6bfc6) Signed-off-by: Paweł Gronowski --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 38bde1c40d0ac..04a48239ce0f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -469,8 +469,6 @@ ENV CONTAINERD_NAMESPACE=moby WORKDIR /go/src/github.com/docker/docker VOLUME /var/lib/docker VOLUME /home/unprivilegeduser/.local/share/docker -COPY --link --from=dockercli /build/ /usr/local/cli -COPY --link --from=dockercli-integration /build/ /usr/local/cli-integration # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] @@ -551,6 +549,8 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \ libsecret-1-dev \ libsystemd-dev \ libudev-dev +COPY --link --from=dockercli /build/ /usr/local/cli +COPY --link --from=dockercli-integration /build/ /usr/local/cli-integration FROM base AS build COPY --from=gowinres /build/ /usr/local/bin/ From aa47b29dbcfc39fd81242c007b302dff7e0394aa Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 31 May 2023 22:56:20 +0200 Subject: [PATCH 072/293] vendor: github.com/moby/swarmkit/v2 v2.0.0-20230531205928-01bb7a41396b - Fix timeouts from very long raft messages - fix: code optimization - update dependencies full diff: https://github.com/moby/swarmkit/compare/75e92ce14ff7...01bb7a41396b57ac1951713b77857f918f4fa84f Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 06aaf87aab77e6f53daceef8dc5678bd4d363f5b) Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 +- .../swarmkit/v2/manager/scheduler/volumes.go | 6 +-- .../v2/manager/state/raft/transport/peer.go | 40 ++++++++++++++++++- .../swarmkit/v2/manager/state/store/memory.go | 7 ++-- vendor/modules.txt | 2 +- 6 files changed, 47 insertions(+), 14 deletions(-) diff --git a/vendor.mod b/vendor.mod index 7f1c45b67e501..adaa62a15113d 100644 --- a/vendor.mod +++ b/vendor.mod @@ -61,7 +61,7 @@ require ( github.com/moby/locker v1.0.1 github.com/moby/patternmatcher v0.5.0 github.com/moby/pubsub v1.0.0 - github.com/moby/swarmkit/v2 v2.0.0-20230406225228-75e92ce14ff7 + github.com/moby/swarmkit/v2 v2.0.0-20230531205928-01bb7a41396b github.com/moby/sys/mount v0.3.3 github.com/moby/sys/mountinfo v0.6.2 github.com/moby/sys/sequential v0.5.0 diff --git a/vendor.sum b/vendor.sum index 8214f95e95a2f..190fe89200ff7 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1053,8 +1053,8 @@ github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/pubsub v1.0.0 h1:jkp/imWsmJz2f6LyFsk7EkVeN2HxR/HTTOY8kHrsxfA= github.com/moby/pubsub v1.0.0/go.mod h1:bXSO+3h5MNXXCaEG+6/NlAIk7MMZbySZlnB+cUQhKKc= -github.com/moby/swarmkit/v2 v2.0.0-20230406225228-75e92ce14ff7 h1:h6NclNly6/B9N4IdM5pcBaq/LkNLuaCmE7B44Vj+pb0= -github.com/moby/swarmkit/v2 v2.0.0-20230406225228-75e92ce14ff7/go.mod h1:P/ha3F7UZMmuUvqrHw9cZK/BjktSngQIgRPiairNHTc= +github.com/moby/swarmkit/v2 v2.0.0-20230531205928-01bb7a41396b h1:w07xyBXYTrihwBqCkuXPLqcQ1a2guqXlRIocU+e9K7A= +github.com/moby/swarmkit/v2 v2.0.0-20230531205928-01bb7a41396b/go.mod h1:Z5i5At5g0zU+ZBWb/95yVwDeNQX8BZmei9ZoYvoVD7g= github.com/moby/sys/mount v0.1.0/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74= github.com/moby/sys/mount v0.1.1/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74= github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs= diff --git a/vendor/github.com/moby/swarmkit/v2/manager/scheduler/volumes.go b/vendor/github.com/moby/swarmkit/v2/manager/scheduler/volumes.go index 9ddba6be16aad..cbd2f74119402 100644 --- a/vendor/github.com/moby/swarmkit/v2/manager/scheduler/volumes.go +++ b/vendor/github.com/moby/swarmkit/v2/manager/scheduler/volumes.go @@ -303,11 +303,7 @@ func (vs *volumeSet) checkVolume(id string, info *NodeInfo, readOnly bool) bool // then, do the quick check of whether this volume is in the topology. if // the volume has an AccessibleTopology, and it does not lie within the // node's topology, then this volume won't fit. - if !IsInTopology(top, vi.volume.VolumeInfo.AccessibleTopology) { - return false - } - - return true + return IsInTopology(top, vi.volume.VolumeInfo.AccessibleTopology) } // hasWriter is a helper function that returns true if at least one task is diff --git a/vendor/github.com/moby/swarmkit/v2/manager/state/raft/transport/peer.go b/vendor/github.com/moby/swarmkit/v2/manager/state/raft/transport/peer.go index 237b871619e45..071f6dc76ff2d 100644 --- a/vendor/github.com/moby/swarmkit/v2/manager/state/raft/transport/peer.go +++ b/vendor/github.com/moby/swarmkit/v2/manager/state/raft/transport/peer.go @@ -196,9 +196,44 @@ func needsSplitting(m *raftpb.Message) bool { } func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error { - ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout) + // These lines used to be in the code, but they've been removed. I'm + // leaving them in in a comment just in case they cause some unforeseen + // breakage later, to show why they were removed. + // + // ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout) + // defer cancel() + // + // Basically, these lines created a timeout that applied not to each chunk + // of a streaming message, but to the whole streaming process. With a + // sufficiently large raft log, the bandwidth on some connections can not + // physically be enough to fit within the default 2 second timeout. + // Further, it seems that because of some gRPC magic, the timeout was + // getting propagated to the stream *server*, meaning it wasn't even the + // sender timing out, it was the receiver. + // + // It should be fine to remove this timeout. The whole purpose of this + // method is to send very large raft messages that could take several + // seconds to send. + + ctx, cancel := context.WithCancel(ctx) defer cancel() + // This is a bootleg watchdog timer. If the timer elapses without something + // being written to the bump channel, it will cancel the context. + // + // We use this because the operations on this stream *must* either time out + // or succeed for raft to function correctly. We can't just time out the + // whole operation, because of the reasons stated above. But we also only + // set the context once, when we create the stream, and so can't set an + // individual timeout for each stream operation. + // + // By doing it as this watchdog-type structure, we can time out individual + // operations by canceling the context on our own terms. + t := time.AfterFunc(p.tr.config.SendTimeout, cancel) + defer t.Stop() + + bump := func() { t.Reset(p.tr.config.SendTimeout) } + var err error var stream api.Raft_StreamRaftMessageClient stream, err = api.NewRaftClient(p.conn()).StreamRaftMessage(ctx) @@ -222,6 +257,9 @@ func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error { stream.CloseAndRecv() break } + + // If the send succeeds, bump the watchdog timer. + bump() } // Finished sending all the messages. diff --git a/vendor/github.com/moby/swarmkit/v2/manager/state/store/memory.go b/vendor/github.com/moby/swarmkit/v2/manager/state/store/memory.go index 1726ac1beeb81..4814e04551fec 100644 --- a/vendor/github.com/moby/swarmkit/v2/manager/state/store/memory.go +++ b/vendor/github.com/moby/swarmkit/v2/manager/state/store/memory.go @@ -16,7 +16,6 @@ import ( gogotypes "github.com/gogo/protobuf/types" memdb "github.com/hashicorp/go-memdb" "github.com/moby/swarmkit/v2/api" - pb "github.com/moby/swarmkit/v2/api" "github.com/moby/swarmkit/v2/manager/state" "github.com/moby/swarmkit/v2/watch" ) @@ -855,8 +854,8 @@ func (tx readTx) find(table string, by By, checkType func(By) error, appendResul } // Save serializes the data in the store. -func (s *MemoryStore) Save(tx ReadTx) (*pb.StoreSnapshot, error) { - var snapshot pb.StoreSnapshot +func (s *MemoryStore) Save(tx ReadTx) (*api.StoreSnapshot, error) { + var snapshot api.StoreSnapshot for _, os := range objectStorers { if err := os.Save(tx, &snapshot); err != nil { return nil, err @@ -868,7 +867,7 @@ func (s *MemoryStore) Save(tx ReadTx) (*pb.StoreSnapshot, error) { // Restore sets the contents of the store to the serialized data in the // argument. -func (s *MemoryStore) Restore(snapshot *pb.StoreSnapshot) error { +func (s *MemoryStore) Restore(snapshot *api.StoreSnapshot) error { return s.updateLocal(func(tx Tx) error { for _, os := range objectStorers { if err := os.Restore(tx, snapshot); err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index 44a2cdff77381..3c0f9754c55a8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -735,7 +735,7 @@ github.com/moby/patternmatcher # github.com/moby/pubsub v1.0.0 ## explicit; go 1.19 github.com/moby/pubsub -# github.com/moby/swarmkit/v2 v2.0.0-20230406225228-75e92ce14ff7 +# github.com/moby/swarmkit/v2 v2.0.0-20230531205928-01bb7a41396b ## explicit; go 1.18 github.com/moby/swarmkit/v2/agent github.com/moby/swarmkit/v2/agent/configs From 8c552012ae128165a4542d7a7aa655fed605f78f Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 2 Jun 2023 18:40:45 -0600 Subject: [PATCH 073/293] contrib/check-config: check for xt_bpf We omit xt_u32 as it's optional; since we will remove support for this module in the future, it's simpler to check for xt_bpf, which will become the new baseline. Related issues: * https://github.com/microsoft/WSL/issues/10029#issuecomment-1574440255 * https://github.com/docker/for-win/issues/13450#issuecomment-1574443139 Signed-off-by: Bjorn Neergaard (cherry picked from commit 1910fdde818e82b28136481a72cd74e4c67f0975) Signed-off-by: Bjorn Neergaard --- contrib/check-config.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/check-config.sh b/contrib/check-config.sh index 10abc43b5f020..0b807ab957b52 100755 --- a/contrib/check-config.sh +++ b/contrib/check-config.sh @@ -220,6 +220,7 @@ check_flags \ VETH BRIDGE BRIDGE_NETFILTER \ IP_NF_FILTER IP_NF_TARGET_MASQUERADE \ NETFILTER_XT_MATCH_ADDRTYPE \ + NETFILTER_XT_MATCH_BPF \ NETFILTER_XT_MATCH_CONNTRACK \ NETFILTER_XT_MATCH_IPVS \ NETFILTER_XT_MARK \ From b3133d7471214ffcbd0e42a16f026bacf4440b5b Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Sun, 4 Jun 2023 13:12:13 -0600 Subject: [PATCH 074/293] contrib/check-config: move xt_bpf check to overlay section Signed-off-by: Bjorn Neergaard (cherry picked from commit 800ea039ec6f64261b709a579afcb31265014ccf) Signed-off-by: Bjorn Neergaard --- contrib/check-config.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/check-config.sh b/contrib/check-config.sh index 0b807ab957b52..b525dd3d0cc06 100755 --- a/contrib/check-config.sh +++ b/contrib/check-config.sh @@ -220,7 +220,6 @@ check_flags \ VETH BRIDGE BRIDGE_NETFILTER \ IP_NF_FILTER IP_NF_TARGET_MASQUERADE \ NETFILTER_XT_MATCH_ADDRTYPE \ - NETFILTER_XT_MATCH_BPF \ NETFILTER_XT_MATCH_CONNTRACK \ NETFILTER_XT_MATCH_IPVS \ NETFILTER_XT_MARK \ @@ -352,7 +351,7 @@ echo " - \"$(wrap_color 'overlay' blue)\":" check_flags VXLAN BRIDGE_VLAN_FILTERING | sed 's/^/ /' echo ' Optional (for encrypted networks):' check_flags CRYPTO CRYPTO_AEAD CRYPTO_GCM CRYPTO_SEQIV CRYPTO_GHASH \ - XFRM XFRM_USER XFRM_ALGO INET_ESP | sed 's/^/ /' + XFRM XFRM_USER XFRM_ALGO INET_ESP NETFILTER_XT_MATCH_BPF | sed 's/^/ /' if [ "$kernelMajor" -lt 5 ] || [ "$kernelMajor" -eq 5 -a "$kernelMinor" -le 3 ]; then check_flags INET_XFRM_MODE_TRANSPORT | sed 's/^/ /' fi From 24c882c3e06f61fa6b66c08101399eec898432ed Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 14 Jun 2023 12:47:05 +0200 Subject: [PATCH 075/293] update go to go1.20.5 go1.20.5 (released 2023-06-06) includes four security fixes to the cmd/go and runtime packages, as well as bug fixes to the compiler, the go command, the runtime, and the crypto/rsa, net, and os packages. See the Go 1.20.5 milestone on our issue tracker for details: https://github.com/golang/go/issues?q=milestone%3AGo1.20.5+label%3ACherryPickApproved full diff: https://github.com/golang/go/compare/go1.20.4...go1.20.5 These minor releases include 3 security fixes following the security policy: - cmd/go: cgo code injection The go command may generate unexpected code at build time when using cgo. This may result in unexpected behavior when running a go program which uses cgo. This may occur when running an untrusted module which contains directories with newline characters in their names. Modules which are retrieved using the go command, i.e. via "go get", are not affected (modules retrieved using GOPATH-mode, i.e. GO111MODULE=off, may be affected). Thanks to Juho Nurminen of Mattermost for reporting this issue. This is CVE-2023-29402 and Go issue https://go.dev/issue/60167. - runtime: unexpected behavior of setuid/setgid binaries The Go runtime didn't act any differently when a binary had the setuid/setgid bit set. On Unix platforms, if a setuid/setgid binary was executed with standard I/O file descriptors closed, opening any files could result in unexpected content being read/written with elevated prilieges. Similarly if a setuid/setgid program was terminated, either via panic or signal, it could leak the contents of its registers. Thanks to Vincent Dehors from Synacktiv for reporting this issue. This is CVE-2023-29403 and Go issue https://go.dev/issue/60272. - cmd/go: improper sanitization of LDFLAGS The go command may execute arbitrary code at build time when using cgo. This may occur when running "go get" on a malicious module, or when running any other command which builds untrusted code. This is can by triggered by linker flags, specified via a "#cgo LDFLAGS" directive. Thanks to Juho Nurminen of Mattermost for reporting this issue. This is CVE-2023-29404 and CVE-2023-29405 and Go issues https://go.dev/issue/60305 and https://go.dev/issue/60306. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 98a44bb18e21c9729575992c1d4a8cbee5a40bb7) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/.windows.yml | 2 +- .github/workflows/test.yml | 2 +- Dockerfile | 2 +- Dockerfile.simple | 2 +- Dockerfile.windows | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml index 8c3df71cbc8b0..a5465dcc200c9 100644 --- a/.github/workflows/.windows.yml +++ b/.github/workflows/.windows.yml @@ -15,7 +15,7 @@ on: default: false env: - GO_VERSION: "1.20.4" + GO_VERSION: "1.20.5" GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 WINDOWS_BASE_IMAGE: mcr.microsoft.com/windows/servercore diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec9e4e68862d8..6556709203bba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ on: pull_request: env: - GO_VERSION: "1.20.4" + GO_VERSION: "1.20.5" GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 ITG_CLI_MATRIX_SIZE: 6 diff --git a/Dockerfile b/Dockerfile index 04a48239ce0f7..1d183ddc5e2e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20.4 +ARG GO_VERSION=1.20.5 ARG BASE_DEBIAN_DISTRO="bullseye" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" ARG XX_VERSION=1.2.1 diff --git a/Dockerfile.simple b/Dockerfile.simple index e0013d6258d09..0431db9b80b20 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -5,7 +5,7 @@ # This represents the bare minimum required to build and test Docker. -ARG GO_VERSION=1.20.4 +ARG GO_VERSION=1.20.5 ARG BASE_DEBIAN_DISTRO="bullseye" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" diff --git a/Dockerfile.windows b/Dockerfile.windows index 43258035d25bf..0383d11ba9919 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -165,7 +165,7 @@ FROM microsoft/windowsservercore # Use PowerShell as the default shell SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -ARG GO_VERSION=1.20.4 +ARG GO_VERSION=1.20.5 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 ARG CONTAINERD_VERSION=v1.7.1 From ae4a10df67295bfcd0ebb62ec219a84e7e882dab Mon Sep 17 00:00:00 2001 From: Jan Garcia Date: Tue, 6 Jun 2023 09:26:27 +0200 Subject: [PATCH 076/293] update RootlessKit to v1.1.1 Signed-off-by: Jan Garcia (cherry picked from commit 0b1c1877c5285fc78c0faa35a040e255d46a40a6) Signed-off-by: Sebastiaan van Stijn --- hack/dockerfile/install/rootlesskit.installer | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/dockerfile/install/rootlesskit.installer b/hack/dockerfile/install/rootlesskit.installer index 32d4fa393d4d7..37090936dada5 100755 --- a/hack/dockerfile/install/rootlesskit.installer +++ b/hack/dockerfile/install/rootlesskit.installer @@ -1,8 +1,8 @@ #!/bin/sh # When updating, also update rootlesskit commit in vendor.mod accordingly -# v1.1.0 -: "${ROOTLESSKIT_VERSION:=6222b477d4c3ce6eea2bcff0586e43c95d1c0bb7}" +# v1.1.1 +: "${ROOTLESSKIT_VERSION:=a2c596ff9b3fddc0c2becb38f2ef4004f15765b5}" install_rootlesskit() { case "$1" in From 1c18ad6ca6d3f2c8c2e8a416c2a24ea970d2e07c Mon Sep 17 00:00:00 2001 From: Jan Garcia Date: Tue, 6 Jun 2023 09:25:59 +0200 Subject: [PATCH 077/293] vendor: github.com/rootless-containers/rootlesskit v1.1.1 Signed-off-by: Jan Garcia (cherry picked from commit 8c4dfc9e6a3eeba376f352e8e3309e0e671afe43) Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 10 +-- vendor.sum | 22 +++--- vendor/github.com/sirupsen/logrus/README.md | 8 ++- vendor/golang.org/x/net/http2/pipe.go | 6 +- vendor/golang.org/x/net/http2/server.go | 7 +- vendor/golang.org/x/net/http2/transport.go | 41 ++++++++--- .../golang.org/x/net/internal/socks/socks.go | 2 +- vendor/golang.org/x/sys/unix/ioctl_signed.go | 70 +++++++++++++++++++ .../sys/unix/{ioctl.go => ioctl_unsigned.go} | 4 +- vendor/golang.org/x/sys/unix/ioctl_zos.go | 12 ++-- vendor/golang.org/x/sys/unix/mkerrors.sh | 5 +- vendor/golang.org/x/sys/unix/syscall_aix.go | 4 +- .../golang.org/x/sys/unix/syscall_aix_ppc.go | 1 - .../x/sys/unix/syscall_aix_ppc64.go | 1 - .../golang.org/x/sys/unix/syscall_darwin.go | 3 +- .../x/sys/unix/syscall_dragonfly.go | 1 - .../golang.org/x/sys/unix/syscall_freebsd.go | 1 - vendor/golang.org/x/sys/unix/syscall_linux.go | 10 ++- .../x/sys/unix/syscall_linux_386.go | 27 ------- .../x/sys/unix/syscall_linux_amd64.go | 1 - .../x/sys/unix/syscall_linux_arm.go | 27 ------- .../x/sys/unix/syscall_linux_arm64.go | 10 --- .../x/sys/unix/syscall_linux_loong64.go | 5 -- .../x/sys/unix/syscall_linux_mips64x.go | 1 - .../x/sys/unix/syscall_linux_mipsx.go | 27 ------- .../x/sys/unix/syscall_linux_ppc.go | 27 ------- .../x/sys/unix/syscall_linux_ppc64x.go | 1 - .../x/sys/unix/syscall_linux_riscv64.go | 1 - .../x/sys/unix/syscall_linux_s390x.go | 1 - .../x/sys/unix/syscall_linux_sparc64.go | 1 - .../golang.org/x/sys/unix/syscall_netbsd.go | 2 - .../golang.org/x/sys/unix/syscall_openbsd.go | 1 - .../golang.org/x/sys/unix/syscall_solaris.go | 21 +++--- vendor/golang.org/x/sys/unix/syscall_unix.go | 7 ++ .../x/sys/unix/syscall_zos_s390x.go | 4 +- .../x/sys/unix/zerrors_darwin_amd64.go | 19 +++++ .../x/sys/unix/zerrors_darwin_arm64.go | 19 +++++ vendor/golang.org/x/sys/unix/zerrors_linux.go | 14 ++++ .../golang.org/x/sys/unix/zsyscall_aix_ppc.go | 15 +--- .../x/sys/unix/zsyscall_aix_ppc64.go | 18 ++--- .../x/sys/unix/zsyscall_aix_ppc64_gc.go | 10 --- .../x/sys/unix/zsyscall_aix_ppc64_gccgo.go | 10 +-- .../x/sys/unix/zsyscall_darwin_amd64.go | 39 +++++++---- .../x/sys/unix/zsyscall_darwin_amd64.s | 11 ++- .../x/sys/unix/zsyscall_darwin_arm64.go | 39 +++++++---- .../x/sys/unix/zsyscall_darwin_arm64.s | 11 ++- .../x/sys/unix/zsyscall_dragonfly_amd64.go | 10 --- .../x/sys/unix/zsyscall_freebsd_386.go | 10 --- .../x/sys/unix/zsyscall_freebsd_amd64.go | 10 --- .../x/sys/unix/zsyscall_freebsd_arm.go | 10 --- .../x/sys/unix/zsyscall_freebsd_arm64.go | 10 --- .../x/sys/unix/zsyscall_freebsd_riscv64.go | 10 --- .../golang.org/x/sys/unix/zsyscall_linux.go | 10 --- .../x/sys/unix/zsyscall_linux_386.go | 10 --- .../x/sys/unix/zsyscall_linux_amd64.go | 10 --- .../x/sys/unix/zsyscall_linux_arm.go | 10 --- .../x/sys/unix/zsyscall_linux_arm64.go | 10 --- .../x/sys/unix/zsyscall_linux_mips.go | 10 --- .../x/sys/unix/zsyscall_linux_mips64.go | 10 --- .../x/sys/unix/zsyscall_linux_mips64le.go | 10 --- .../x/sys/unix/zsyscall_linux_mipsle.go | 10 --- .../x/sys/unix/zsyscall_linux_ppc.go | 10 --- .../x/sys/unix/zsyscall_linux_ppc64.go | 10 --- .../x/sys/unix/zsyscall_linux_ppc64le.go | 10 --- .../x/sys/unix/zsyscall_linux_riscv64.go | 10 --- .../x/sys/unix/zsyscall_linux_s390x.go | 10 --- .../x/sys/unix/zsyscall_linux_sparc64.go | 10 --- .../x/sys/unix/zsyscall_netbsd_386.go | 10 --- .../x/sys/unix/zsyscall_netbsd_amd64.go | 10 --- .../x/sys/unix/zsyscall_netbsd_arm.go | 10 --- .../x/sys/unix/zsyscall_netbsd_arm64.go | 10 --- .../x/sys/unix/zsyscall_openbsd_386.go | 14 ---- .../x/sys/unix/zsyscall_openbsd_386.s | 5 -- .../x/sys/unix/zsyscall_openbsd_amd64.go | 14 ---- .../x/sys/unix/zsyscall_openbsd_amd64.s | 5 -- .../x/sys/unix/zsyscall_openbsd_arm.go | 14 ---- .../x/sys/unix/zsyscall_openbsd_arm.s | 5 -- .../x/sys/unix/zsyscall_openbsd_arm64.go | 14 ---- .../x/sys/unix/zsyscall_openbsd_arm64.s | 5 -- .../x/sys/unix/zsyscall_openbsd_mips64.go | 14 ---- .../x/sys/unix/zsyscall_openbsd_mips64.s | 5 -- .../x/sys/unix/zsyscall_openbsd_ppc64.go | 14 ---- .../x/sys/unix/zsyscall_openbsd_ppc64.s | 6 -- .../x/sys/unix/zsyscall_openbsd_riscv64.go | 14 ---- .../x/sys/unix/zsyscall_openbsd_riscv64.s | 5 -- .../x/sys/unix/zsyscall_solaris_amd64.go | 17 +---- .../x/sys/unix/zsyscall_zos_s390x.go | 4 +- .../x/sys/unix/ztypes_darwin_amd64.go | 11 +++ .../x/sys/unix/ztypes_darwin_arm64.go | 11 +++ .../golang.org/x/sys/windows/env_windows.go | 6 +- .../golang.org/x/sys/windows/exec_windows.go | 7 +- vendor/golang.org/x/sys/windows/service.go | 7 ++ .../x/sys/windows/svc/mgr/service.go | 57 +++++++++++++-- .../golang.org/x/sys/windows/svc/service.go | 9 +++ .../golang.org/x/sys/windows/types_windows.go | 10 ++- .../x/sys/windows/zsyscall_windows.go | 9 +++ vendor/modules.txt | 10 +-- 97 files changed, 414 insertions(+), 705 deletions(-) create mode 100644 vendor/golang.org/x/sys/unix/ioctl_signed.go rename vendor/golang.org/x/sys/unix/{ioctl.go => ioctl_unsigned.go} (92%) diff --git a/vendor.mod b/vendor.mod index 7f1c45b67e501..f21c5c2540d07 100644 --- a/vendor.mod +++ b/vendor.mod @@ -77,8 +77,8 @@ require ( github.com/pelletier/go-toml v1.9.5 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 - github.com/rootless-containers/rootlesskit v1.1.0 - github.com/sirupsen/logrus v1.9.0 + github.com/rootless-containers/rootlesskit v1.1.1 + github.com/sirupsen/logrus v1.9.2 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/tonistiigi/fsutil v0.0.0-20230105215944-fb433841cbfa @@ -87,10 +87,10 @@ require ( github.com/vishvananda/netlink v1.2.1-beta.2 github.com/vishvananda/netns v0.0.2 go.etcd.io/bbolt v1.3.7 - golang.org/x/net v0.8.0 + golang.org/x/net v0.10.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.6.0 - golang.org/x/text v0.8.0 + golang.org/x/sys v0.8.0 + golang.org/x/text v0.9.0 golang.org/x/time v0.3.0 google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 google.golang.org/grpc v1.50.1 diff --git a/vendor.sum b/vendor.sum index 8214f95e95a2f..dbce2db84b597 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1262,8 +1262,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rootless-containers/rootlesskit v1.1.0 h1:cRaRIYxY8oce4eE/zeAUZhgKu/4tU1p9YHN4+suwV7M= -github.com/rootless-containers/rootlesskit v1.1.0/go.mod h1:H+o9ndNe7tS91WqU0/+vpvc+VaCd7TCIWaJjnV0ujUo= +github.com/rootless-containers/rootlesskit v1.1.1 h1:F5psKWoWY9/VjZ3ifVcaosjvFZJOagX85U22M0/EQZE= +github.com/rootless-containers/rootlesskit v1.1.1/go.mod h1:UD5GoA3dqKCJrnvnhVgQQnweMF2qZnf9KLw8EewcMZI= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -1302,8 +1302,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= @@ -1661,8 +1661,8 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1835,12 +1835,12 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1851,8 +1851,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/vendor/github.com/sirupsen/logrus/README.md b/vendor/github.com/sirupsen/logrus/README.md index b042c896f25b4..d1d4a85fd752d 100644 --- a/vendor/github.com/sirupsen/logrus/README.md +++ b/vendor/github.com/sirupsen/logrus/README.md @@ -9,7 +9,7 @@ the last thing you want from your Logging library (again...). This does not mean Logrus is dead. Logrus will continue to be maintained for security, (backwards compatible) bug fixes, and performance (where we are -limited by the interface). +limited by the interface). I believe Logrus' biggest contribution is to have played a part in today's widespread use of structured logging in Golang. There doesn't seem to be a @@ -43,7 +43,7 @@ plain text): With `log.SetFormatter(&log.JSONFormatter{})`, for easy parsing by logstash or Splunk: -```json +```text {"animal":"walrus","level":"info","msg":"A group of walrus emerges from the ocean","size":10,"time":"2014-03-10 19:57:38.562264131 -0400 EDT"} @@ -99,7 +99,7 @@ time="2015-03-26T01:27:38-04:00" level=fatal method=github.com/sirupsen/arcticcr ``` Note that this does add measurable overhead - the cost will depend on the version of Go, but is between 20 and 40% in recent tests with 1.6 and 1.7. You can validate this in your -environment via benchmarks: +environment via benchmarks: ``` go test -bench=.*CallerTracing ``` @@ -317,6 +317,8 @@ log.SetLevel(log.InfoLevel) It may be useful to set `log.Level = logrus.DebugLevel` in a debug or verbose environment if your application has that. +Note: If you want different log levels for global (`log.SetLevel(...)`) and syslog logging, please check the [syslog hook README](hooks/syslog/README.md#different-log-levels-for-local-and-remote-logging). + #### Entries Besides the fields added with `WithField` or `WithFields` some fields are diff --git a/vendor/golang.org/x/net/http2/pipe.go b/vendor/golang.org/x/net/http2/pipe.go index c15b8a7719b5c..684d984fd96af 100644 --- a/vendor/golang.org/x/net/http2/pipe.go +++ b/vendor/golang.org/x/net/http2/pipe.go @@ -88,13 +88,9 @@ func (p *pipe) Write(d []byte) (n int, err error) { p.c.L = &p.mu } defer p.c.Signal() - if p.err != nil { + if p.err != nil || p.breakErr != nil { return 0, errClosedPipeWrite } - if p.breakErr != nil { - p.unread += len(d) - return len(d), nil // discard when there is no reader - } return p.b.Write(d) } diff --git a/vendor/golang.org/x/net/http2/server.go b/vendor/golang.org/x/net/http2/server.go index 8cb14f3c97f53..cd057f3982480 100644 --- a/vendor/golang.org/x/net/http2/server.go +++ b/vendor/golang.org/x/net/http2/server.go @@ -1822,15 +1822,18 @@ func (sc *serverConn) processData(f *DataFrame) error { } if len(data) > 0 { + st.bodyBytes += int64(len(data)) wrote, err := st.body.Write(data) if err != nil { + // The handler has closed the request body. + // Return the connection-level flow control for the discarded data, + // but not the stream-level flow control. sc.sendWindowUpdate(nil, int(f.Length)-wrote) - return sc.countError("body_write_err", streamError(id, ErrCodeStreamClosed)) + return nil } if wrote != len(data) { panic("internal error: bad Writer") } - st.bodyBytes += int64(len(data)) } // Return any padded flow control now, since we won't diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index 05ba23d3d9886..ac90a2631c9e6 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -560,10 +560,11 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res traceGotConn(req, cc, reused) res, err := cc.RoundTrip(req) if err != nil && retry <= 6 { + roundTripErr := err if req, err = shouldRetryRequest(req, err); err == nil { // After the first retry, do exponential backoff with 10% jitter. if retry == 0 { - t.vlogf("RoundTrip retrying after failure: %v", err) + t.vlogf("RoundTrip retrying after failure: %v", roundTripErr) continue } backoff := float64(uint(1) << (uint(retry) - 1)) @@ -572,7 +573,7 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res timer := backoffNewTimer(d) select { case <-timer.C: - t.vlogf("RoundTrip retrying after failure: %v", err) + t.vlogf("RoundTrip retrying after failure: %v", roundTripErr) continue case <-req.Context().Done(): timer.Stop() @@ -1265,6 +1266,27 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { return res, nil } + cancelRequest := func(cs *clientStream, err error) error { + cs.cc.mu.Lock() + defer cs.cc.mu.Unlock() + cs.abortStreamLocked(err) + if cs.ID != 0 { + // This request may have failed because of a problem with the connection, + // or for some unrelated reason. (For example, the user might have canceled + // the request without waiting for a response.) Mark the connection as + // not reusable, since trying to reuse a dead connection is worse than + // unnecessarily creating a new one. + // + // If cs.ID is 0, then the request was never allocated a stream ID and + // whatever went wrong was unrelated to the connection. We might have + // timed out waiting for a stream slot when StrictMaxConcurrentStreams + // is set, for example, in which case retrying on a different connection + // will not help. + cs.cc.doNotReuse = true + } + return err + } + for { select { case <-cs.respHeaderRecv: @@ -1279,15 +1301,12 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { return handleResponseHeaders() default: waitDone() - return nil, cs.abortErr + return nil, cancelRequest(cs, cs.abortErr) } case <-ctx.Done(): - err := ctx.Err() - cs.abortStream(err) - return nil, err + return nil, cancelRequest(cs, ctx.Err()) case <-cs.reqCancel: - cs.abortStream(errRequestCanceled) - return nil, errRequestCanceled + return nil, cancelRequest(cs, errRequestCanceled) } } } @@ -2555,6 +2574,9 @@ func (b transportResponseBody) Close() error { cs := b.cs cc := cs.cc + cs.bufPipe.BreakWithError(errClosedResponseBody) + cs.abortStream(errClosedResponseBody) + unread := cs.bufPipe.Len() if unread > 0 { cc.mu.Lock() @@ -2573,9 +2595,6 @@ func (b transportResponseBody) Close() error { cc.wmu.Unlock() } - cs.bufPipe.BreakWithError(errClosedResponseBody) - cs.abortStream(errClosedResponseBody) - select { case <-cs.donec: case <-cs.ctx.Done(): diff --git a/vendor/golang.org/x/net/internal/socks/socks.go b/vendor/golang.org/x/net/internal/socks/socks.go index 97db2340ec970..84fcc32b634b8 100644 --- a/vendor/golang.org/x/net/internal/socks/socks.go +++ b/vendor/golang.org/x/net/internal/socks/socks.go @@ -289,7 +289,7 @@ func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, case AuthMethodNotRequired: return nil case AuthMethodUsernamePassword: - if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) == 0 || len(up.Password) > 255 { + if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 { return errors.New("invalid username/password") } b := []byte{authUsernamePasswordVersion} diff --git a/vendor/golang.org/x/sys/unix/ioctl_signed.go b/vendor/golang.org/x/sys/unix/ioctl_signed.go new file mode 100644 index 0000000000000..7def9580e6f85 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/ioctl_signed.go @@ -0,0 +1,70 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || solaris +// +build aix solaris + +package unix + +import ( + "unsafe" +) + +// ioctl itself should not be exposed directly, but additional get/set +// functions for specific types are permissible. + +// IoctlSetInt performs an ioctl operation which sets an integer value +// on fd, using the specified request number. +func IoctlSetInt(fd int, req int, value int) error { + return ioctl(fd, req, uintptr(value)) +} + +// IoctlSetPointerInt performs an ioctl operation which sets an +// integer value on fd, using the specified request number. The ioctl +// argument is called with a pointer to the integer value, rather than +// passing the integer value directly. +func IoctlSetPointerInt(fd int, req int, value int) error { + v := int32(value) + return ioctlPtr(fd, req, unsafe.Pointer(&v)) +} + +// IoctlSetWinsize performs an ioctl on fd with a *Winsize argument. +// +// To change fd's window size, the req argument should be TIOCSWINSZ. +func IoctlSetWinsize(fd int, req int, value *Winsize) error { + // TODO: if we get the chance, remove the req parameter and + // hardcode TIOCSWINSZ. + return ioctlPtr(fd, req, unsafe.Pointer(value)) +} + +// IoctlSetTermios performs an ioctl on fd with a *Termios. +// +// The req value will usually be TCSETA or TIOCSETA. +func IoctlSetTermios(fd int, req int, value *Termios) error { + // TODO: if we get the chance, remove the req parameter. + return ioctlPtr(fd, req, unsafe.Pointer(value)) +} + +// IoctlGetInt performs an ioctl operation which gets an integer value +// from fd, using the specified request number. +// +// A few ioctl requests use the return value as an output parameter; +// for those, IoctlRetInt should be used instead of this function. +func IoctlGetInt(fd int, req int) (int, error) { + var value int + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) + return value, err +} + +func IoctlGetWinsize(fd int, req int) (*Winsize, error) { + var value Winsize + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) + return &value, err +} + +func IoctlGetTermios(fd int, req int) (*Termios, error) { + var value Termios + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) + return &value, err +} diff --git a/vendor/golang.org/x/sys/unix/ioctl.go b/vendor/golang.org/x/sys/unix/ioctl_unsigned.go similarity index 92% rename from vendor/golang.org/x/sys/unix/ioctl.go rename to vendor/golang.org/x/sys/unix/ioctl_unsigned.go index 7ce8dd406fff5..649913d1ea71a 100644 --- a/vendor/golang.org/x/sys/unix/ioctl.go +++ b/vendor/golang.org/x/sys/unix/ioctl_unsigned.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build aix || darwin || dragonfly || freebsd || hurd || linux || netbsd || openbsd || solaris -// +build aix darwin dragonfly freebsd hurd linux netbsd openbsd solaris +//go:build darwin || dragonfly || freebsd || hurd || linux || netbsd || openbsd +// +build darwin dragonfly freebsd hurd linux netbsd openbsd package unix diff --git a/vendor/golang.org/x/sys/unix/ioctl_zos.go b/vendor/golang.org/x/sys/unix/ioctl_zos.go index 6532f09af2e33..cdc21bf76dcbb 100644 --- a/vendor/golang.org/x/sys/unix/ioctl_zos.go +++ b/vendor/golang.org/x/sys/unix/ioctl_zos.go @@ -17,14 +17,14 @@ import ( // IoctlSetInt performs an ioctl operation which sets an integer value // on fd, using the specified request number. -func IoctlSetInt(fd int, req uint, value int) error { +func IoctlSetInt(fd int, req int, value int) error { return ioctl(fd, req, uintptr(value)) } // IoctlSetWinsize performs an ioctl on fd with a *Winsize argument. // // To change fd's window size, the req argument should be TIOCSWINSZ. -func IoctlSetWinsize(fd int, req uint, value *Winsize) error { +func IoctlSetWinsize(fd int, req int, value *Winsize) error { // TODO: if we get the chance, remove the req parameter and // hardcode TIOCSWINSZ. return ioctlPtr(fd, req, unsafe.Pointer(value)) @@ -33,7 +33,7 @@ func IoctlSetWinsize(fd int, req uint, value *Winsize) error { // IoctlSetTermios performs an ioctl on fd with a *Termios. // // The req value is expected to be TCSETS, TCSETSW, or TCSETSF -func IoctlSetTermios(fd int, req uint, value *Termios) error { +func IoctlSetTermios(fd int, req int, value *Termios) error { if (req != TCSETS) && (req != TCSETSW) && (req != TCSETSF) { return ENOSYS } @@ -47,13 +47,13 @@ func IoctlSetTermios(fd int, req uint, value *Termios) error { // // A few ioctl requests use the return value as an output parameter; // for those, IoctlRetInt should be used instead of this function. -func IoctlGetInt(fd int, req uint) (int, error) { +func IoctlGetInt(fd int, req int) (int, error) { var value int err := ioctlPtr(fd, req, unsafe.Pointer(&value)) return value, err } -func IoctlGetWinsize(fd int, req uint) (*Winsize, error) { +func IoctlGetWinsize(fd int, req int) (*Winsize, error) { var value Winsize err := ioctlPtr(fd, req, unsafe.Pointer(&value)) return &value, err @@ -62,7 +62,7 @@ func IoctlGetWinsize(fd int, req uint) (*Winsize, error) { // IoctlGetTermios performs an ioctl on fd with a *Termios. // // The req value is expected to be TCGETS -func IoctlGetTermios(fd int, req uint) (*Termios, error) { +func IoctlGetTermios(fd int, req int) (*Termios, error) { var value Termios if req != TCGETS { return &value, ENOSYS diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh index 7456d9ddde16b..be0423e6856b3 100644 --- a/vendor/golang.org/x/sys/unix/mkerrors.sh +++ b/vendor/golang.org/x/sys/unix/mkerrors.sh @@ -66,6 +66,7 @@ includes_Darwin=' #include #include #include +#include #include #include #include @@ -203,6 +204,7 @@ struct ltchars { #include #include #include +#include #include #include #include @@ -517,10 +519,11 @@ ccflags="$@" $2 ~ /^LOCK_(SH|EX|NB|UN)$/ || $2 ~ /^LO_(KEY|NAME)_SIZE$/ || $2 ~ /^LOOP_(CLR|CTL|GET|SET)_/ || - $2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|TCP|MCAST|EVFILT|NOTE|SHUT|PROT|MAP|MFD|T?PACKET|MSG|SCM|MCL|DT|MADV|PR|LOCAL|TCPOPT)_/ || + $2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|TCP|MCAST|EVFILT|NOTE|SHUT|PROT|MAP|MFD|T?PACKET|MSG|SCM|MCL|DT|MADV|PR|LOCAL|TCPOPT|UDP)_/ || $2 ~ /^NFC_(GENL|PROTO|COMM|RF|SE|DIRECTION|LLCP|SOCKPROTO)_/ || $2 ~ /^NFC_.*_(MAX)?SIZE$/ || $2 ~ /^RAW_PAYLOAD_/ || + $2 ~ /^[US]F_/ || $2 ~ /^TP_STATUS_/ || $2 ~ /^FALLOC_/ || $2 ~ /^ICMPV?6?_(FILTER|SEC)/ || diff --git a/vendor/golang.org/x/sys/unix/syscall_aix.go b/vendor/golang.org/x/sys/unix/syscall_aix.go index d9f5544ccf454..c406ae00f417f 100644 --- a/vendor/golang.org/x/sys/unix/syscall_aix.go +++ b/vendor/golang.org/x/sys/unix/syscall_aix.go @@ -408,8 +408,8 @@ func (w WaitStatus) CoreDump() bool { return w&0x80 == 0x80 } func (w WaitStatus) TrapCause() int { return -1 } -//sys ioctl(fd int, req uint, arg uintptr) (err error) -//sys ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) = ioctl +//sys ioctl(fd int, req int, arg uintptr) (err error) +//sys ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) = ioctl // fcntl must never be called with cmd=F_DUP2FD because it doesn't work on AIX // There is no way to create a custom fcntl and to keep //sys fcntl easily, diff --git a/vendor/golang.org/x/sys/unix/syscall_aix_ppc.go b/vendor/golang.org/x/sys/unix/syscall_aix_ppc.go index e92a0be1630c7..f2871fa953512 100644 --- a/vendor/golang.org/x/sys/unix/syscall_aix_ppc.go +++ b/vendor/golang.org/x/sys/unix/syscall_aix_ppc.go @@ -8,7 +8,6 @@ package unix //sysnb Getrlimit(resource int, rlim *Rlimit) (err error) = getrlimit64 -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) = setrlimit64 //sys Seek(fd int, offset int64, whence int) (off int64, err error) = lseek64 //sys mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int64) (xaddr uintptr, err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_aix_ppc64.go b/vendor/golang.org/x/sys/unix/syscall_aix_ppc64.go index 16eed17098e5f..75718ec0f19b5 100644 --- a/vendor/golang.org/x/sys/unix/syscall_aix_ppc64.go +++ b/vendor/golang.org/x/sys/unix/syscall_aix_ppc64.go @@ -8,7 +8,6 @@ package unix //sysnb Getrlimit(resource int, rlim *Rlimit) (err error) -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) //sys Seek(fd int, offset int64, whence int) (off int64, err error) = lseek //sys mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int64) (xaddr uintptr, err error) = mmap64 diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin.go b/vendor/golang.org/x/sys/unix/syscall_darwin.go index 7064d6ebab6a8..206921504cb6b 100644 --- a/vendor/golang.org/x/sys/unix/syscall_darwin.go +++ b/vendor/golang.org/x/sys/unix/syscall_darwin.go @@ -613,6 +613,7 @@ func SysctlKinfoProcSlice(name string, args ...int) ([]KinfoProc, error) { //sys Rmdir(path string) (err error) //sys Seek(fd int, offset int64, whence int) (newoffset int64, err error) = SYS_LSEEK //sys Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err error) +//sys Setattrlist(path string, attrlist *Attrlist, attrBuf []byte, options int) (err error) //sys Setegid(egid int) (err error) //sysnb Seteuid(euid int) (err error) //sysnb Setgid(gid int) (err error) @@ -622,7 +623,6 @@ func SysctlKinfoProcSlice(name string, args ...int) ([]KinfoProc, error) { //sys Setprivexec(flag int) (err error) //sysnb Setregid(rgid int, egid int) (err error) //sysnb Setreuid(ruid int, euid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setsid() (pid int, err error) //sysnb Settimeofday(tp *Timeval) (err error) //sysnb Setuid(uid int) (err error) @@ -676,7 +676,6 @@ func SysctlKinfoProcSlice(name string, args ...int) ([]KinfoProc, error) { // Kqueue_from_portset_np // Kqueue_portset // Getattrlist -// Setattrlist // Getdirentriesattr // Searchfs // Delete diff --git a/vendor/golang.org/x/sys/unix/syscall_dragonfly.go b/vendor/golang.org/x/sys/unix/syscall_dragonfly.go index 221efc26bcdc6..d4ce988e72fbd 100644 --- a/vendor/golang.org/x/sys/unix/syscall_dragonfly.go +++ b/vendor/golang.org/x/sys/unix/syscall_dragonfly.go @@ -326,7 +326,6 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e //sysnb Setreuid(ruid int, euid int) (err error) //sysnb Setresgid(rgid int, egid int, sgid int) (err error) //sysnb Setresuid(ruid int, euid int, suid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setsid() (pid int, err error) //sysnb Settimeofday(tp *Timeval) (err error) //sysnb Setuid(uid int) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_freebsd.go b/vendor/golang.org/x/sys/unix/syscall_freebsd.go index 5bdde03e4a847..afb10106f6e6b 100644 --- a/vendor/golang.org/x/sys/unix/syscall_freebsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_freebsd.go @@ -433,7 +433,6 @@ func Dup3(oldfd, newfd, flags int) error { //sysnb Setreuid(ruid int, euid int) (err error) //sysnb Setresgid(rgid int, egid int, sgid int) (err error) //sysnb Setresuid(ruid int, euid int, suid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setsid() (pid int, err error) //sysnb Settimeofday(tp *Timeval) (err error) //sysnb Setuid(uid int) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go index 9735331530ac3..fbaeb5fff1484 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux.go @@ -1873,7 +1873,6 @@ func Getpgrp() (pid int) { //sys OpenTree(dfd int, fileName string, flags uint) (r int, err error) //sys PerfEventOpen(attr *PerfEventAttr, pid int, cpu int, groupFd int, flags int) (fd int, err error) //sys PivotRoot(newroot string, putold string) (err error) = SYS_PIVOT_ROOT -//sysnb Prlimit(pid int, resource int, newlimit *Rlimit, old *Rlimit) (err error) = SYS_PRLIMIT64 //sys Prctl(option int, arg2 uintptr, arg3 uintptr, arg4 uintptr, arg5 uintptr) (err error) //sys Pselect(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timespec, sigmask *Sigset_t) (n int, err error) = SYS_PSELECT6 //sys read(fd int, p []byte) (n int, err error) @@ -1887,6 +1886,15 @@ func Getpgrp() (pid int) { //sysnb Settimeofday(tv *Timeval) (err error) //sys Setns(fd int, nstype int) (err error) +//go:linkname syscall_prlimit syscall.prlimit +func syscall_prlimit(pid, resource int, newlimit, old *syscall.Rlimit) error + +func Prlimit(pid, resource int, newlimit, old *Rlimit) error { + // Just call the syscall version, because as of Go 1.21 + // it will affect starting a new process. + return syscall_prlimit(pid, resource, (*syscall.Rlimit)(newlimit), (*syscall.Rlimit)(old)) +} + // PrctlRetInt performs a prctl operation specified by option and further // optional arguments arg2 through arg5 depending on option. It returns a // non-negative integer that is returned by the prctl syscall. diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_386.go b/vendor/golang.org/x/sys/unix/syscall_linux_386.go index ff5b5899d6db3..c7d9945ea19af 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_386.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_386.go @@ -97,33 +97,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { return } -//sysnb setrlimit(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func Setrlimit(resource int, rlim *Rlimit) (err error) { - err = Prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit(resource, &rl) -} - func Seek(fd int, offset int64, whence int) (newoffset int64, err error) { newoffset, errno := seek(fd, offset, whence) if errno != 0 { diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_amd64.go b/vendor/golang.org/x/sys/unix/syscall_linux_amd64.go index 9b2703532989a..5b21fcfd75393 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_amd64.go @@ -46,7 +46,6 @@ func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys setfsgid(gid int) (prev int, err error) //sys setfsuid(uid int) (prev int, err error) -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_arm.go b/vendor/golang.org/x/sys/unix/syscall_linux_arm.go index 856ad1d635cfd..da2986415ae22 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_arm.go @@ -171,33 +171,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { return } -//sysnb setrlimit(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func Setrlimit(resource int, rlim *Rlimit) (err error) { - err = Prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit(resource, &rl) -} - func (r *PtraceRegs) PC() uint64 { return uint64(r.Uregs[15]) } func (r *PtraceRegs) SetPC(pc uint64) { r.Uregs[15] = uint32(pc) } diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go b/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go index 6422704bc52aa..a81f5742b8a58 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_arm64.go @@ -39,7 +39,6 @@ func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys setfsgid(gid int) (prev int, err error) //sys setfsuid(uid int) (prev int, err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) @@ -143,15 +142,6 @@ func Getrlimit(resource int, rlim *Rlimit) error { return getrlimit(resource, rlim) } -// Setrlimit prefers the prlimit64 system call. See issue 38604. -func Setrlimit(resource int, rlim *Rlimit) error { - err := Prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - return setrlimit(resource, rlim) -} - func (r *PtraceRegs) PC() uint64 { return r.Pc } func (r *PtraceRegs) SetPC(pc uint64) { r.Pc = pc } diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go b/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go index 59dab510e97ca..69d2d7c3db7a4 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_loong64.go @@ -126,11 +126,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { return } -func Setrlimit(resource int, rlim *Rlimit) (err error) { - err = Prlimit(0, resource, rlim, nil) - return -} - func futimesat(dirfd int, path string, tv *[2]Timeval) (err error) { if tv == nil { return utimensat(dirfd, path, nil, 0) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go b/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go index bfef09a39eb06..76d564095ef49 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go @@ -37,7 +37,6 @@ func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys setfsgid(gid int) (prev int, err error) //sys setfsuid(uid int) (prev int, err error) -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Statfs(path string, buf *Statfs_t) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_mipsx.go b/vendor/golang.org/x/sys/unix/syscall_linux_mipsx.go index ab302509663e6..aae7f0ffd3fcf 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_mipsx.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_mipsx.go @@ -151,33 +151,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { return } -//sysnb setrlimit(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func Setrlimit(resource int, rlim *Rlimit) (err error) { - err = Prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit(resource, &rl) -} - func (r *PtraceRegs) PC() uint64 { return r.Epc } func (r *PtraceRegs) SetPC(pc uint64) { r.Epc = pc } diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_ppc.go b/vendor/golang.org/x/sys/unix/syscall_linux_ppc.go index eac1cf1acc86c..66eff19a320bd 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_ppc.go @@ -159,33 +159,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { return } -//sysnb setrlimit(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func Setrlimit(resource int, rlim *Rlimit) (err error) { - err = Prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit(resource, &rl) -} - func (r *PtraceRegs) PC() uint32 { return r.Nip } func (r *PtraceRegs) SetPC(pc uint32) { r.Nip = pc } diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_ppc64x.go b/vendor/golang.org/x/sys/unix/syscall_linux_ppc64x.go index 4df56616b8f1a..806aa2574d8df 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_ppc64x.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_ppc64x.go @@ -34,7 +34,6 @@ package unix //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys setfsgid(gid int) (prev int, err error) //sys setfsuid(uid int) (prev int, err error) -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Stat(path string, stat *Stat_t) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go b/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go index 5f4243dea2c3b..35851ef70b8d1 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_riscv64.go @@ -38,7 +38,6 @@ func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys setfsgid(gid int) (prev int, err error) //sys setfsuid(uid int) (prev int, err error) -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_s390x.go b/vendor/golang.org/x/sys/unix/syscall_linux_s390x.go index d0a7d4066851f..2f89e8f5defe3 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_s390x.go @@ -34,7 +34,6 @@ import ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys setfsgid(gid int) (prev int, err error) //sys setfsuid(uid int) (prev int, err error) -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Stat(path string, stat *Stat_t) (err error) //sys Statfs(path string, buf *Statfs_t) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_sparc64.go b/vendor/golang.org/x/sys/unix/syscall_linux_sparc64.go index f5c793be26d4a..7ca064ae76495 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_sparc64.go @@ -31,7 +31,6 @@ package unix //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys setfsgid(gid int) (prev int, err error) //sys setfsuid(uid int) (prev int, err error) -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Stat(path string, stat *Stat_t) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_netbsd.go b/vendor/golang.org/x/sys/unix/syscall_netbsd.go index e66865dccbe6c..018d7d47822f7 100644 --- a/vendor/golang.org/x/sys/unix/syscall_netbsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_netbsd.go @@ -340,7 +340,6 @@ func Statvfs(path string, buf *Statvfs_t) (err error) { //sys Setpriority(which int, who int, prio int) (err error) //sysnb Setregid(rgid int, egid int) (err error) //sysnb Setreuid(ruid int, euid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setsid() (pid int, err error) //sysnb Settimeofday(tp *Timeval) (err error) //sysnb Setuid(uid int) (err error) @@ -501,7 +500,6 @@ func Statvfs(path string, buf *Statvfs_t) (err error) { // compat_43_osendmsg // compat_43_osethostid // compat_43_osethostname -// compat_43_osetrlimit // compat_43_osigblock // compat_43_osigsetmask // compat_43_osigstack diff --git a/vendor/golang.org/x/sys/unix/syscall_openbsd.go b/vendor/golang.org/x/sys/unix/syscall_openbsd.go index 5e9de23ae372b..f9c7a9663c6a6 100644 --- a/vendor/golang.org/x/sys/unix/syscall_openbsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_openbsd.go @@ -294,7 +294,6 @@ func Uname(uname *Utsname) error { //sysnb Setreuid(ruid int, euid int) (err error) //sysnb Setresgid(rgid int, egid int, sgid int) (err error) //sysnb Setresuid(ruid int, euid int, suid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setrtable(rtable int) (err error) //sysnb Setsid() (pid int, err error) //sysnb Settimeofday(tp *Timeval) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_solaris.go b/vendor/golang.org/x/sys/unix/syscall_solaris.go index d3444b64d6d1e..b600a289d3386 100644 --- a/vendor/golang.org/x/sys/unix/syscall_solaris.go +++ b/vendor/golang.org/x/sys/unix/syscall_solaris.go @@ -545,24 +545,24 @@ func Minor(dev uint64) uint32 { * Expose the ioctl function */ -//sys ioctlRet(fd int, req uint, arg uintptr) (ret int, err error) = libc.ioctl -//sys ioctlPtrRet(fd int, req uint, arg unsafe.Pointer) (ret int, err error) = libc.ioctl +//sys ioctlRet(fd int, req int, arg uintptr) (ret int, err error) = libc.ioctl +//sys ioctlPtrRet(fd int, req int, arg unsafe.Pointer) (ret int, err error) = libc.ioctl -func ioctl(fd int, req uint, arg uintptr) (err error) { +func ioctl(fd int, req int, arg uintptr) (err error) { _, err = ioctlRet(fd, req, arg) return err } -func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { +func ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) { _, err = ioctlPtrRet(fd, req, arg) return err } -func IoctlSetTermio(fd int, req uint, value *Termio) error { +func IoctlSetTermio(fd int, req int, value *Termio) error { return ioctlPtr(fd, req, unsafe.Pointer(value)) } -func IoctlGetTermio(fd int, req uint) (*Termio, error) { +func IoctlGetTermio(fd int, req int) (*Termio, error) { var value Termio err := ioctlPtr(fd, req, unsafe.Pointer(&value)) return &value, err @@ -665,7 +665,6 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e //sys Setpriority(which int, who int, prio int) (err error) //sysnb Setregid(rgid int, egid int) (err error) //sysnb Setreuid(ruid int, euid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setsid() (pid int, err error) //sysnb Setuid(uid int) (err error) //sys Shutdown(s int, how int) (err error) = libsocket.shutdown @@ -1080,11 +1079,11 @@ func Getmsg(fd int, cl []byte, data []byte) (retCl []byte, retData []byte, flags return retCl, retData, flags, nil } -func IoctlSetIntRetInt(fd int, req uint, arg int) (int, error) { +func IoctlSetIntRetInt(fd int, req int, arg int) (int, error) { return ioctlRet(fd, req, uintptr(arg)) } -func IoctlSetString(fd int, req uint, val string) error { +func IoctlSetString(fd int, req int, val string) error { bs := make([]byte, len(val)+1) copy(bs[:len(bs)-1], val) err := ioctlPtr(fd, req, unsafe.Pointer(&bs[0])) @@ -1120,7 +1119,7 @@ func (l *Lifreq) GetLifruUint() uint { return *(*uint)(unsafe.Pointer(&l.Lifru[0])) } -func IoctlLifreq(fd int, req uint, l *Lifreq) error { +func IoctlLifreq(fd int, req int, l *Lifreq) error { return ioctlPtr(fd, req, unsafe.Pointer(l)) } @@ -1131,6 +1130,6 @@ func (s *Strioctl) SetInt(i int) { s.Dp = (*int8)(unsafe.Pointer(&i)) } -func IoctlSetStrioctlRetInt(fd int, req uint, s *Strioctl) (int, error) { +func IoctlSetStrioctlRetInt(fd int, req int, s *Strioctl) (int, error) { return ioctlPtrRet(fd, req, unsafe.Pointer(s)) } diff --git a/vendor/golang.org/x/sys/unix/syscall_unix.go b/vendor/golang.org/x/sys/unix/syscall_unix.go index 00f0aa3758892..8e48c29ec332d 100644 --- a/vendor/golang.org/x/sys/unix/syscall_unix.go +++ b/vendor/golang.org/x/sys/unix/syscall_unix.go @@ -587,3 +587,10 @@ func emptyIovecs(iov []Iovec) bool { } return true } + +// Setrlimit sets a resource limit. +func Setrlimit(resource int, rlim *Rlimit) error { + // Just call the syscall version, because as of Go 1.21 + // it will affect starting a new process. + return syscall.Setrlimit(resource, (*syscall.Rlimit)(rlim)) +} diff --git a/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go b/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go index b295497ae476a..d3d49ec3ed759 100644 --- a/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go +++ b/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go @@ -212,8 +212,8 @@ func (cmsg *Cmsghdr) SetLen(length int) { //sys sendmsg(s int, msg *Msghdr, flags int) (n int, err error) = SYS___SENDMSG_A //sys mmap(addr uintptr, length uintptr, prot int, flag int, fd int, pos int64) (ret uintptr, err error) = SYS_MMAP //sys munmap(addr uintptr, length uintptr) (err error) = SYS_MUNMAP -//sys ioctl(fd int, req uint, arg uintptr) (err error) = SYS_IOCTL -//sys ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) = SYS_IOCTL +//sys ioctl(fd int, req int, arg uintptr) (err error) = SYS_IOCTL +//sys ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) = SYS_IOCTL //sys Access(path string, mode uint32) (err error) = SYS___ACCESS_A //sys Chdir(path string) (err error) = SYS___CHDIR_A diff --git a/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go index 476a1c7e77c52..1430076271501 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go @@ -1270,6 +1270,16 @@ const ( SEEK_END = 0x2 SEEK_HOLE = 0x3 SEEK_SET = 0x0 + SF_APPEND = 0x40000 + SF_ARCHIVED = 0x10000 + SF_DATALESS = 0x40000000 + SF_FIRMLINK = 0x800000 + SF_IMMUTABLE = 0x20000 + SF_NOUNLINK = 0x100000 + SF_RESTRICTED = 0x80000 + SF_SETTABLE = 0x3fff0000 + SF_SUPPORTED = 0x9f0000 + SF_SYNTHETIC = 0xc0000000 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 @@ -1543,6 +1553,15 @@ const ( TIOCTIMESTAMP = 0x40107459 TIOCUCNTL = 0x80047466 TOSTOP = 0x400000 + UF_APPEND = 0x4 + UF_COMPRESSED = 0x20 + UF_DATAVAULT = 0x80 + UF_HIDDEN = 0x8000 + UF_IMMUTABLE = 0x2 + UF_NODUMP = 0x1 + UF_OPAQUE = 0x8 + UF_SETTABLE = 0xffff + UF_TRACKED = 0x40 VDISCARD = 0xf VDSUSP = 0xb VEOF = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go index e36f5178d6008..ab044a74274f0 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go @@ -1270,6 +1270,16 @@ const ( SEEK_END = 0x2 SEEK_HOLE = 0x3 SEEK_SET = 0x0 + SF_APPEND = 0x40000 + SF_ARCHIVED = 0x10000 + SF_DATALESS = 0x40000000 + SF_FIRMLINK = 0x800000 + SF_IMMUTABLE = 0x20000 + SF_NOUNLINK = 0x100000 + SF_RESTRICTED = 0x80000 + SF_SETTABLE = 0x3fff0000 + SF_SUPPORTED = 0x9f0000 + SF_SYNTHETIC = 0xc0000000 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 @@ -1543,6 +1553,15 @@ const ( TIOCTIMESTAMP = 0x40107459 TIOCUCNTL = 0x80047466 TOSTOP = 0x400000 + UF_APPEND = 0x4 + UF_COMPRESSED = 0x20 + UF_DATAVAULT = 0x80 + UF_HIDDEN = 0x8000 + UF_IMMUTABLE = 0x2 + UF_NODUMP = 0x1 + UF_OPAQUE = 0x8 + UF_SETTABLE = 0xffff + UF_TRACKED = 0x40 VDISCARD = 0xf VDSUSP = 0xb VEOF = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux.go b/vendor/golang.org/x/sys/unix/zerrors_linux.go index 398c37e52d6b6..de936b677b6aa 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux.go @@ -2967,6 +2967,7 @@ const ( SOL_TCP = 0x6 SOL_TIPC = 0x10f SOL_TLS = 0x11a + SOL_UDP = 0x11 SOL_X25 = 0x106 SOL_XDP = 0x11b SOMAXCONN = 0x1000 @@ -3251,6 +3252,19 @@ const ( TRACEFS_MAGIC = 0x74726163 TS_COMM_LEN = 0x20 UDF_SUPER_MAGIC = 0x15013346 + UDP_CORK = 0x1 + UDP_ENCAP = 0x64 + UDP_ENCAP_ESPINUDP = 0x2 + UDP_ENCAP_ESPINUDP_NON_IKE = 0x1 + UDP_ENCAP_GTP0 = 0x4 + UDP_ENCAP_GTP1U = 0x5 + UDP_ENCAP_L2TPINUDP = 0x3 + UDP_GRO = 0x68 + UDP_NO_CHECK6_RX = 0x66 + UDP_NO_CHECK6_TX = 0x65 + UDP_SEGMENT = 0x67 + UDP_V4_FLOW = 0x2 + UDP_V6_FLOW = 0x6 UMOUNT_NOFOLLOW = 0x8 USBDEVICE_SUPER_MAGIC = 0x9fa2 UTIME_NOW = 0x3fffffff diff --git a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc.go b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc.go index ef9dcd1bef8c9..9a257219d7067 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc.go @@ -124,7 +124,6 @@ int utime(uintptr_t, uintptr_t); unsigned long long getsystemcfg(int); int umount(uintptr_t); int getrlimit64(int, uintptr_t); -int setrlimit64(int, uintptr_t); long long lseek64(int, long long, int); uintptr_t mmap(uintptr_t, uintptr_t, int, int, int, long long); @@ -213,7 +212,7 @@ func wait4(pid Pid_t, status *_C_int, options int, rusage *Rusage) (wpid Pid_t, // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctl(fd int, req uint, arg uintptr) (err error) { +func ioctl(fd int, req int, arg uintptr) (err error) { r0, er := C.ioctl(C.int(fd), C.int(req), C.uintptr_t(arg)) if r0 == -1 && er != nil { err = er @@ -223,7 +222,7 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { +func ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) { r0, er := C.ioctl(C.int(fd), C.int(req), C.uintptr_t(uintptr(arg))) if r0 == -1 && er != nil { err = er @@ -1464,16 +1463,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - r0, er := C.setrlimit64(C.int(resource), C.uintptr_t(uintptr(unsafe.Pointer(rlim)))) - if r0 == -1 && er != nil { - err = er - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Seek(fd int, offset int64, whence int) (off int64, err error) { r0, er := C.lseek64(C.int(fd), C.longlong(offset), C.int(whence)) off = int64(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64.go b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64.go index f86a94592348b..6de80c20cf2a2 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64.go @@ -93,8 +93,8 @@ func wait4(pid Pid_t, status *_C_int, options int, rusage *Rusage) (wpid Pid_t, // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctl(fd int, req uint, arg uintptr) (err error) { - _, e1 := callioctl(fd, int(req), arg) +func ioctl(fd int, req int, arg uintptr) (err error) { + _, e1 := callioctl(fd, req, arg) if e1 != 0 { err = errnoErr(e1) } @@ -103,8 +103,8 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { - _, e1 := callioctl_ptr(fd, int(req), arg) +func ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) { + _, e1 := callioctl_ptr(fd, req, arg) if e1 != 0 { err = errnoErr(e1) } @@ -1422,16 +1422,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, e1 := callsetrlimit(resource, uintptr(unsafe.Pointer(rlim))) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Seek(fd int, offset int64, whence int) (off int64, err error) { r0, e1 := calllseek(fd, offset, whence) off = int64(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gc.go b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gc.go index d32a84cae27c1..c4d50ae5005c2 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gc.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gc.go @@ -124,7 +124,6 @@ import ( //go:cgo_import_dynamic libc_getsystemcfg getsystemcfg "libc.a/shr_64.o" //go:cgo_import_dynamic libc_umount umount "libc.a/shr_64.o" //go:cgo_import_dynamic libc_getrlimit getrlimit "libc.a/shr_64.o" -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.a/shr_64.o" //go:cgo_import_dynamic libc_lseek lseek "libc.a/shr_64.o" //go:cgo_import_dynamic libc_mmap64 mmap64 "libc.a/shr_64.o" @@ -242,7 +241,6 @@ import ( //go:linkname libc_getsystemcfg libc_getsystemcfg //go:linkname libc_umount libc_umount //go:linkname libc_getrlimit libc_getrlimit -//go:linkname libc_setrlimit libc_setrlimit //go:linkname libc_lseek libc_lseek //go:linkname libc_mmap64 libc_mmap64 @@ -363,7 +361,6 @@ var ( libc_getsystemcfg, libc_umount, libc_getrlimit, - libc_setrlimit, libc_lseek, libc_mmap64 syscallFunc ) @@ -1179,13 +1176,6 @@ func callgetrlimit(resource int, rlim uintptr) (r1 uintptr, e1 Errno) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func callsetrlimit(resource int, rlim uintptr) (r1 uintptr, e1 Errno) { - r1, _, e1 = rawSyscall6(uintptr(unsafe.Pointer(&libc_setrlimit)), 2, uintptr(resource), rlim, 0, 0, 0, 0) - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func calllseek(fd int, offset int64, whence int) (r1 uintptr, e1 Errno) { r1, _, e1 = syscall6(uintptr(unsafe.Pointer(&libc_lseek)), 3, uintptr(fd), uintptr(offset), uintptr(whence), 0, 0, 0) return diff --git a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gccgo.go b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gccgo.go index d7d8baf819c07..6903d3b09e3da 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gccgo.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gccgo.go @@ -123,7 +123,6 @@ int utime(uintptr_t, uintptr_t); unsigned long long getsystemcfg(int); int umount(uintptr_t); int getrlimit(int, uintptr_t); -int setrlimit(int, uintptr_t); long long lseek(int, long long, int); uintptr_t mmap64(uintptr_t, uintptr_t, int, int, int, long long); @@ -131,6 +130,7 @@ uintptr_t mmap64(uintptr_t, uintptr_t, int, int, int, long long); import "C" import ( "syscall" + "unsafe" ) // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT @@ -1055,14 +1055,6 @@ func callgetrlimit(resource int, rlim uintptr) (r1 uintptr, e1 Errno) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func callsetrlimit(resource int, rlim uintptr) (r1 uintptr, e1 Errno) { - r1 = uintptr(C.setrlimit(C.int(resource), C.uintptr_t(rlim))) - e1 = syscall.GetErrno() - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func calllseek(fd int, offset int64, whence int) (r1 uintptr, e1 Errno) { r1 = uintptr(C.lseek(C.int(fd), C.longlong(offset), C.int(whence))) e1 = syscall.GetErrno() diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go index a29ffdd566db1..4037ccf7a940e 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go @@ -1992,6 +1992,31 @@ var libc_select_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Setattrlist(path string, attrlist *Attrlist, attrBuf []byte, options int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(attrBuf) > 0 { + _p1 = unsafe.Pointer(&attrBuf[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall_syscall6(libc_setattrlist_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(attrlist)), uintptr(_p1), uintptr(len(attrBuf)), uintptr(options), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_setattrlist_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setattrlist setattrlist "/usr/lib/libSystem.B.dylib" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Setegid(egid int) (err error) { _, _, e1 := syscall_syscall(libc_setegid_trampoline_addr, uintptr(egid), 0, 0) if e1 != 0 { @@ -2123,20 +2148,6 @@ var libc_setreuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "/usr/lib/libSystem.B.dylib" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := syscall_rawSyscall(libc_setsid_trampoline_addr, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s index 95fe4c0eb962d..4baaed0bc12ca 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s @@ -705,6 +705,11 @@ TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_select_trampoline_addr(SB), RODATA, $8 DATA ·libc_select_trampoline_addr(SB)/8, $libc_select_trampoline<>(SB) +TEXT libc_setattrlist_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setattrlist(SB) +GLOBL ·libc_setattrlist_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setattrlist_trampoline_addr(SB)/8, $libc_setattrlist_trampoline<>(SB) + TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) @@ -759,12 +764,6 @@ TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setreuid_trampoline_addr(SB)/8, $libc_setreuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go index 2fd4590bb7866..51d6f3fb25681 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go @@ -1992,6 +1992,31 @@ var libc_select_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Setattrlist(path string, attrlist *Attrlist, attrBuf []byte, options int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(attrBuf) > 0 { + _p1 = unsafe.Pointer(&attrBuf[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall_syscall6(libc_setattrlist_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(attrlist)), uintptr(_p1), uintptr(len(attrBuf)), uintptr(options), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_setattrlist_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setattrlist setattrlist "/usr/lib/libSystem.B.dylib" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Setegid(egid int) (err error) { _, _, e1 := syscall_syscall(libc_setegid_trampoline_addr, uintptr(egid), 0, 0) if e1 != 0 { @@ -2123,20 +2148,6 @@ var libc_setreuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "/usr/lib/libSystem.B.dylib" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := syscall_rawSyscall(libc_setsid_trampoline_addr, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s index efa5b4c987c56..c3b82c03793fa 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s @@ -705,6 +705,11 @@ TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_select_trampoline_addr(SB), RODATA, $8 DATA ·libc_select_trampoline_addr(SB)/8, $libc_select_trampoline<>(SB) +TEXT libc_setattrlist_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setattrlist(SB) +GLOBL ·libc_setattrlist_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setattrlist_trampoline_addr(SB)/8, $libc_setattrlist_trampoline<>(SB) + TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) @@ -759,12 +764,6 @@ TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setreuid_trampoline_addr(SB)/8, $libc_setreuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go index 3b85134707efd..0eabac7ade213 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go @@ -1410,16 +1410,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go index 1129065624e58..ee313eb0073b3 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go @@ -1645,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go index 55f5abfe599c3..4c986e448ee9d 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go @@ -1645,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go index d39651c2b586b..555216944a0e0 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go @@ -1645,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go index ddb7408680118..67a226fbf5e32 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go @@ -1645,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go index 09a53a616c050..f0b9ddaaa262c 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go @@ -1645,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux.go b/vendor/golang.org/x/sys/unix/zsyscall_linux.go index 430cb24de7e0e..da63d9d7822ff 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux.go @@ -1346,16 +1346,6 @@ func PivotRoot(newroot string, putold string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Prlimit(pid int, resource int, newlimit *Rlimit, old *Rlimit) (err error) { - _, _, e1 := RawSyscall6(SYS_PRLIMIT64, uintptr(pid), uintptr(resource), uintptr(unsafe.Pointer(newlimit)), uintptr(unsafe.Pointer(old)), 0, 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Prctl(option int, arg2 uintptr, arg3 uintptr, arg4 uintptr, arg5 uintptr) (err error) { _, _, e1 := Syscall6(SYS_PRCTL, uintptr(option), uintptr(arg2), uintptr(arg3), uintptr(arg4), uintptr(arg5), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_386.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_386.go index c81b0ad47772d..07b549cc25e85 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_386.go @@ -411,16 +411,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func futimesat(dirfd int, path string, times *[2]Timeval) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go index 2206bce7f4dda..5f481bf83f46a 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go @@ -334,16 +334,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go index edf6b39f1615e..824cd52c7fae4 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go @@ -578,16 +578,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func armSyncFileRange(fd int, flags int, off int64, n int64) (err error) { _, _, e1 := Syscall6(SYS_ARM_SYNC_FILE_RANGE, uintptr(fd), uintptr(flags), uintptr(off), uintptr(off>>32), uintptr(n), uintptr(n>>32)) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go index 190609f2140d4..e77aecfe98535 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go @@ -289,16 +289,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go index 5f984cbb1ca74..961a3afb7b71b 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go @@ -644,16 +644,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Alarm(seconds uint) (remaining uint, err error) { r0, _, e1 := Syscall(SYS_ALARM, uintptr(seconds), 0, 0) remaining = uint(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go index 46fc380a40e54..ed05005e91b69 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go @@ -278,16 +278,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go index cbd0d4dadbadd..d365b718f3014 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go @@ -278,16 +278,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go index 0c13d15f07cfd..c3f1b8bbde01a 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go @@ -644,16 +644,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Alarm(seconds uint) (remaining uint, err error) { r0, _, e1 := Syscall(SYS_ALARM, uintptr(seconds), 0, 0) remaining = uint(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc.go index e01432aed51f7..a6574cf98b16e 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc.go @@ -624,16 +624,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func syncFileRange2(fd int, flags int, off int64, n int64) (err error) { _, _, e1 := Syscall6(SYS_SYNC_FILE_RANGE2, uintptr(fd), uintptr(flags), uintptr(off>>32), uintptr(off), uintptr(n>>32), uintptr(n)) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go index 13c7ee7baff6c..f40990264f497 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go @@ -349,16 +349,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go index 02d0c0fd61ecd..9dfcc29974f47 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go @@ -349,16 +349,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_riscv64.go index 9fee3b1d23960..0b29239583b95 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_riscv64.go @@ -269,16 +269,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go index 647bbfecd6aa8..6cde32237dc8e 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go @@ -319,16 +319,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) { r0, _, e1 := Syscall6(SYS_SPLICE, uintptr(rfd), uintptr(unsafe.Pointer(roff)), uintptr(wfd), uintptr(unsafe.Pointer(woff)), uintptr(len), uintptr(flags)) n = int64(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go index ada057f89144b..5253d65bf1b96 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go @@ -329,16 +329,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go index 8e1d9c8f66639..cdb2af5ae0f4a 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go @@ -1607,16 +1607,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go index 21c6950400e30..9d25f76b0bfdc 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go @@ -1607,16 +1607,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go index 298168f90a17b..d3f8035169f06 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go @@ -1607,16 +1607,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go index 68b8bd492fec5..887188a529e28 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go @@ -1607,16 +1607,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go index 0b0f910e1ab9c..6699a783e1f0b 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go @@ -1894,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s index 087444250c9a4..04f0de34b2e59 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s @@ -573,11 +573,6 @@ TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setresuid_trampoline_addr(SB)/4, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $4 -DATA ·libc_setrlimit_trampoline_addr(SB)/4, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $4 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go index 48ff5de75b55f..1e775fe05718c 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go @@ -1894,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s index 5782cd1084470..27b6f4df74f1c 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s @@ -573,11 +573,6 @@ TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $8 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go index 2452a641dae7b..7f6427899a5be 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go @@ -1894,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s index cf310420c942d..b797045fd2d1d 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s @@ -573,11 +573,6 @@ TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setresuid_trampoline_addr(SB)/4, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $4 -DATA ·libc_setrlimit_trampoline_addr(SB)/4, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $4 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go index 5e35600a60c3c..756ef7b173620 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go @@ -1894,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s index 484bb42e0a89f..a871266221e47 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s @@ -573,11 +573,6 @@ TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $8 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go index b04cef1a19885..7bc2e24eb95f5 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go @@ -1894,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s index 55af27263ad73..05d4bffd791ea 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s @@ -573,11 +573,6 @@ TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $8 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go index 47a07ee0c2748..739be6217a376 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go @@ -1894,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s index 4028255b0d5bb..74a25f8d64380 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s @@ -687,12 +687,6 @@ TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - CALL libc_setrlimit(SB) - RET -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 CALL libc_setrtable(SB) RET diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go index 573378fdb96f0..7d95a1978033c 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go @@ -1894,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s index e1fbd4dfa8c87..990be2457404c 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s @@ -573,11 +573,6 @@ TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $8 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go index 4873a1e5d3e9c..609d1c598a899 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go @@ -110,7 +110,6 @@ import ( //go:cgo_import_dynamic libc_setpriority setpriority "libc.so" //go:cgo_import_dynamic libc_setregid setregid "libc.so" //go:cgo_import_dynamic libc_setreuid setreuid "libc.so" -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" //go:cgo_import_dynamic libc_setsid setsid "libc.so" //go:cgo_import_dynamic libc_setuid setuid "libc.so" //go:cgo_import_dynamic libc_shutdown shutdown "libsocket.so" @@ -250,7 +249,6 @@ import ( //go:linkname procSetpriority libc_setpriority //go:linkname procSetregid libc_setregid //go:linkname procSetreuid libc_setreuid -//go:linkname procSetrlimit libc_setrlimit //go:linkname procSetsid libc_setsid //go:linkname procSetuid libc_setuid //go:linkname procshutdown libc_shutdown @@ -391,7 +389,6 @@ var ( procSetpriority, procSetregid, procSetreuid, - procSetrlimit, procSetsid, procSetuid, procshutdown, @@ -646,7 +643,7 @@ func __minor(version int, dev uint64) (val uint) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctlRet(fd int, req uint, arg uintptr) (ret int, err error) { +func ioctlRet(fd int, req int, arg uintptr) (ret int, err error) { r0, _, e1 := sysvicall6(uintptr(unsafe.Pointer(&procioctl)), 3, uintptr(fd), uintptr(req), uintptr(arg), 0, 0, 0) ret = int(r0) if e1 != 0 { @@ -657,7 +654,7 @@ func ioctlRet(fd int, req uint, arg uintptr) (ret int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctlPtrRet(fd int, req uint, arg unsafe.Pointer) (ret int, err error) { +func ioctlPtrRet(fd int, req int, arg unsafe.Pointer) (ret int, err error) { r0, _, e1 := sysvicall6(uintptr(unsafe.Pointer(&procioctl)), 3, uintptr(fd), uintptr(req), uintptr(arg), 0, 0, 0) ret = int(r0) if e1 != 0 { @@ -1650,16 +1647,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := rawSysvicall6(uintptr(unsafe.Pointer(&procSetrlimit)), 2, uintptr(which), uintptr(unsafe.Pointer(lim)), 0, 0, 0, 0) - if e1 != 0 { - err = e1 - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := rawSysvicall6(uintptr(unsafe.Pointer(&procSetsid)), 0, 0, 0, 0, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_zos_s390x.go b/vendor/golang.org/x/sys/unix/zsyscall_zos_s390x.go index 07bfe2ef9ad07..c31681743c74c 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_zos_s390x.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_zos_s390x.go @@ -257,7 +257,7 @@ func munmap(addr uintptr, length uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctl(fd int, req uint, arg uintptr) (err error) { +func ioctl(fd int, req int, arg uintptr) (err error) { _, _, e1 := syscall_syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) if e1 != 0 { err = errnoErr(e1) @@ -267,7 +267,7 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { +func ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) { _, _, e1 := syscall_syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) if e1 != 0 { err = errnoErr(e1) diff --git a/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go index e2a64f0991a00..690cefc3d06f1 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go @@ -151,6 +151,16 @@ type Dirent struct { _ [3]byte } +type Attrlist struct { + Bitmapcount uint16 + Reserved uint16 + Commonattr uint32 + Volattr uint32 + Dirattr uint32 + Fileattr uint32 + Forkattr uint32 +} + const ( PathMax = 0x400 ) @@ -610,6 +620,7 @@ const ( AT_REMOVEDIR = 0x80 AT_SYMLINK_FOLLOW = 0x40 AT_SYMLINK_NOFOLLOW = 0x20 + AT_EACCESS = 0x10 ) type PollFd struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go index 34aa775219f04..5bffc10eac09a 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go @@ -151,6 +151,16 @@ type Dirent struct { _ [3]byte } +type Attrlist struct { + Bitmapcount uint16 + Reserved uint16 + Commonattr uint32 + Volattr uint32 + Dirattr uint32 + Fileattr uint32 + Forkattr uint32 +} + const ( PathMax = 0x400 ) @@ -610,6 +620,7 @@ const ( AT_REMOVEDIR = 0x80 AT_SYMLINK_FOLLOW = 0x40 AT_SYMLINK_NOFOLLOW = 0x20 + AT_EACCESS = 0x10 ) type PollFd struct { diff --git a/vendor/golang.org/x/sys/windows/env_windows.go b/vendor/golang.org/x/sys/windows/env_windows.go index 92ac05ff4ea60..b8ad192506890 100644 --- a/vendor/golang.org/x/sys/windows/env_windows.go +++ b/vendor/golang.org/x/sys/windows/env_windows.go @@ -37,14 +37,14 @@ func (token Token) Environ(inheritExisting bool) (env []string, err error) { return nil, err } defer DestroyEnvironmentBlock(block) - blockp := uintptr(unsafe.Pointer(block)) + blockp := unsafe.Pointer(block) for { - entry := UTF16PtrToString((*uint16)(unsafe.Pointer(blockp))) + entry := UTF16PtrToString((*uint16)(blockp)) if len(entry) == 0 { break } env = append(env, entry) - blockp += 2 * (uintptr(len(entry)) + 1) + blockp = unsafe.Add(blockp, 2*(len(entry)+1)) } return env, nil } diff --git a/vendor/golang.org/x/sys/windows/exec_windows.go b/vendor/golang.org/x/sys/windows/exec_windows.go index 75980fd44ad79..a52e0331d8bcd 100644 --- a/vendor/golang.org/x/sys/windows/exec_windows.go +++ b/vendor/golang.org/x/sys/windows/exec_windows.go @@ -95,12 +95,17 @@ func ComposeCommandLine(args []string) string { // DecomposeCommandLine breaks apart its argument command line into unescaped parts using CommandLineToArgv, // as gathered from GetCommandLine, QUERY_SERVICE_CONFIG's BinaryPathName argument, or elsewhere that // command lines are passed around. +// DecomposeCommandLine returns error if commandLine contains NUL. func DecomposeCommandLine(commandLine string) ([]string, error) { if len(commandLine) == 0 { return []string{}, nil } + utf16CommandLine, err := UTF16FromString(commandLine) + if err != nil { + return nil, errorspkg.New("string with NUL passed to DecomposeCommandLine") + } var argc int32 - argv, err := CommandLineToArgv(StringToUTF16Ptr(commandLine), &argc) + argv, err := CommandLineToArgv(&utf16CommandLine[0], &argc) if err != nil { return nil, err } diff --git a/vendor/golang.org/x/sys/windows/service.go b/vendor/golang.org/x/sys/windows/service.go index f8deca8397ae6..c964b6848d4fe 100644 --- a/vendor/golang.org/x/sys/windows/service.go +++ b/vendor/golang.org/x/sys/windows/service.go @@ -141,6 +141,12 @@ const ( SERVICE_DYNAMIC_INFORMATION_LEVEL_START_REASON = 1 ) +type ENUM_SERVICE_STATUS struct { + ServiceName *uint16 + DisplayName *uint16 + ServiceStatus SERVICE_STATUS +} + type SERVICE_STATUS struct { ServiceType uint32 CurrentState uint32 @@ -245,3 +251,4 @@ type QUERY_SERVICE_LOCK_STATUS struct { //sys UnsubscribeServiceChangeNotifications(subscription uintptr) = sechost.UnsubscribeServiceChangeNotifications? //sys RegisterServiceCtrlHandlerEx(serviceName *uint16, handlerProc uintptr, context uintptr) (handle Handle, err error) = advapi32.RegisterServiceCtrlHandlerExW //sys QueryServiceDynamicInformation(service Handle, infoLevel uint32, dynamicInfo unsafe.Pointer) (err error) = advapi32.QueryServiceDynamicInformation? +//sys EnumDependentServices(service Handle, activityState uint32, services *ENUM_SERVICE_STATUS, buffSize uint32, bytesNeeded *uint32, servicesReturned *uint32) (err error) = advapi32.EnumDependentServicesW diff --git a/vendor/golang.org/x/sys/windows/svc/mgr/service.go b/vendor/golang.org/x/sys/windows/svc/mgr/service.go index 0623fc0b0290b..be3d151a3f56d 100644 --- a/vendor/golang.org/x/sys/windows/svc/mgr/service.go +++ b/vendor/golang.org/x/sys/windows/svc/mgr/service.go @@ -15,8 +15,6 @@ import ( "golang.org/x/sys/windows/svc" ) -// TODO(brainman): Use EnumDependentServices to enumerate dependent services. - // Service is used to access Windows service. type Service struct { Name string @@ -47,17 +45,25 @@ func (s *Service) Start(args ...string) error { return windows.StartService(s.Handle, uint32(len(args)), p) } -// Control sends state change request c to the service s. +// Control sends state change request c to the service s. It returns the most +// recent status the service reported to the service control manager, and an +// error if the state change request was not accepted. +// Note that the returned service status is only set if the status change +// request succeeded, or if it failed with error ERROR_INVALID_SERVICE_CONTROL, +// ERROR_SERVICE_CANNOT_ACCEPT_CTRL, or ERROR_SERVICE_NOT_ACTIVE. func (s *Service) Control(c svc.Cmd) (svc.Status, error) { var t windows.SERVICE_STATUS err := windows.ControlService(s.Handle, uint32(c), &t) - if err != nil { + if err != nil && + err != windows.ERROR_INVALID_SERVICE_CONTROL && + err != windows.ERROR_SERVICE_CANNOT_ACCEPT_CTRL && + err != windows.ERROR_SERVICE_NOT_ACTIVE { return svc.Status{}, err } return svc.Status{ State: svc.State(t.CurrentState), Accepts: svc.Accepted(t.ControlsAccepted), - }, nil + }, err } // Query returns current status of service s. @@ -76,3 +82,44 @@ func (s *Service) Query() (svc.Status, error) { ServiceSpecificExitCode: t.ServiceSpecificExitCode, }, nil } + +// ListDependentServices returns the names of the services dependent on service s, which match the given status. +func (s *Service) ListDependentServices(status svc.ActivityStatus) ([]string, error) { + var bytesNeeded, returnedServiceCount uint32 + var services []windows.ENUM_SERVICE_STATUS + for { + var servicesPtr *windows.ENUM_SERVICE_STATUS + if len(services) > 0 { + servicesPtr = &services[0] + } + allocatedBytes := uint32(len(services)) * uint32(unsafe.Sizeof(windows.ENUM_SERVICE_STATUS{})) + err := windows.EnumDependentServices(s.Handle, uint32(status), servicesPtr, allocatedBytes, &bytesNeeded, + &returnedServiceCount) + if err == nil { + break + } + if err != syscall.ERROR_MORE_DATA { + return nil, err + } + if bytesNeeded <= allocatedBytes { + return nil, err + } + // ERROR_MORE_DATA indicates the provided buffer was too small, run the call again after resizing the buffer + requiredSliceLen := bytesNeeded / uint32(unsafe.Sizeof(windows.ENUM_SERVICE_STATUS{})) + if bytesNeeded%uint32(unsafe.Sizeof(windows.ENUM_SERVICE_STATUS{})) != 0 { + requiredSliceLen += 1 + } + services = make([]windows.ENUM_SERVICE_STATUS, requiredSliceLen) + } + if returnedServiceCount == 0 { + return nil, nil + } + + // The slice mutated by EnumDependentServices may have a length greater than returnedServiceCount, any elements + // past that should be ignored. + var dependents []string + for i := 0; i < int(returnedServiceCount); i++ { + dependents = append(dependents, windows.UTF16PtrToString(services[i].ServiceName)) + } + return dependents, nil +} diff --git a/vendor/golang.org/x/sys/windows/svc/service.go b/vendor/golang.org/x/sys/windows/svc/service.go index 806baa055f6e4..2b4a7bc6c2510 100644 --- a/vendor/golang.org/x/sys/windows/svc/service.go +++ b/vendor/golang.org/x/sys/windows/svc/service.go @@ -68,6 +68,15 @@ const ( AcceptPreShutdown = Accepted(windows.SERVICE_ACCEPT_PRESHUTDOWN) ) +// ActivityStatus allows for services to be selected based on active and inactive categories of service state. +type ActivityStatus uint32 + +const ( + Active = ActivityStatus(windows.SERVICE_ACTIVE) + Inactive = ActivityStatus(windows.SERVICE_INACTIVE) + AnyActivity = ActivityStatus(windows.SERVICE_STATE_ALL) +) + // Status combines State and Accepted commands to fully describe running service. type Status struct { State State diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go index 857acf1032d9f..88e62a63851b6 100644 --- a/vendor/golang.org/x/sys/windows/types_windows.go +++ b/vendor/golang.org/x/sys/windows/types_windows.go @@ -2220,19 +2220,23 @@ type JOBOBJECT_BASIC_UI_RESTRICTIONS struct { } const ( - // JobObjectInformationClass + // JobObjectInformationClass for QueryInformationJobObject and SetInformationJobObject JobObjectAssociateCompletionPortInformation = 7 + JobObjectBasicAccountingInformation = 1 + JobObjectBasicAndIoAccountingInformation = 8 JobObjectBasicLimitInformation = 2 + JobObjectBasicProcessIdList = 3 JobObjectBasicUIRestrictions = 4 JobObjectCpuRateControlInformation = 15 JobObjectEndOfJobTimeInformation = 6 JobObjectExtendedLimitInformation = 9 JobObjectGroupInformation = 11 JobObjectGroupInformationEx = 14 - JobObjectLimitViolationInformation2 = 35 + JobObjectLimitViolationInformation = 13 + JobObjectLimitViolationInformation2 = 34 JobObjectNetRateControlInformation = 32 JobObjectNotificationLimitInformation = 12 - JobObjectNotificationLimitInformation2 = 34 + JobObjectNotificationLimitInformation2 = 33 JobObjectSecurityLimitInformation = 5 ) diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go index 6d2a268534d79..a81ea2c700193 100644 --- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go +++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go @@ -86,6 +86,7 @@ var ( procDeleteService = modadvapi32.NewProc("DeleteService") procDeregisterEventSource = modadvapi32.NewProc("DeregisterEventSource") procDuplicateTokenEx = modadvapi32.NewProc("DuplicateTokenEx") + procEnumDependentServicesW = modadvapi32.NewProc("EnumDependentServicesW") procEnumServicesStatusExW = modadvapi32.NewProc("EnumServicesStatusExW") procEqualSid = modadvapi32.NewProc("EqualSid") procFreeSid = modadvapi32.NewProc("FreeSid") @@ -734,6 +735,14 @@ func DuplicateTokenEx(existingToken Token, desiredAccess uint32, tokenAttributes return } +func EnumDependentServices(service Handle, activityState uint32, services *ENUM_SERVICE_STATUS, buffSize uint32, bytesNeeded *uint32, servicesReturned *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEnumDependentServicesW.Addr(), 6, uintptr(service), uintptr(activityState), uintptr(unsafe.Pointer(services)), uintptr(buffSize), uintptr(unsafe.Pointer(bytesNeeded)), uintptr(unsafe.Pointer(servicesReturned))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func EnumServicesStatusEx(mgr Handle, infoLevel uint32, serviceType uint32, serviceState uint32, services *byte, bufSize uint32, bytesNeeded *uint32, servicesReturned *uint32, resumeHandle *uint32, groupName *uint16) (err error) { r1, _, e1 := syscall.Syscall12(procEnumServicesStatusExW.Addr(), 10, uintptr(mgr), uintptr(infoLevel), uintptr(serviceType), uintptr(serviceState), uintptr(unsafe.Pointer(services)), uintptr(bufSize), uintptr(unsafe.Pointer(bytesNeeded)), uintptr(unsafe.Pointer(servicesReturned)), uintptr(unsafe.Pointer(resumeHandle)), uintptr(unsafe.Pointer(groupName)), 0, 0) if r1 == 0 { diff --git a/vendor/modules.txt b/vendor/modules.txt index 44a2cdff77381..4701379a1650b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -881,7 +881,7 @@ github.com/prometheus/common/model github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/rootless-containers/rootlesskit v1.1.0 +# github.com/rootless-containers/rootlesskit v1.1.1 ## explicit; go 1.19 github.com/rootless-containers/rootlesskit/pkg/api github.com/rootless-containers/rootlesskit/pkg/api/client @@ -896,7 +896,7 @@ github.com/secure-systems-lab/go-securesystemslib/dsse # github.com/shibumi/go-pathspec v1.3.0 ## explicit; go 1.17 github.com/shibumi/go-pathspec -# github.com/sirupsen/logrus v1.9.0 +# github.com/sirupsen/logrus v1.9.2 ## explicit; go 1.13 github.com/sirupsen/logrus # github.com/spdx/tools-golang v0.3.1-0.20230104082527-d6f58551be3f @@ -1079,7 +1079,7 @@ golang.org/x/crypto/pkcs12/internal/rc2 golang.org/x/crypto/salsa20/salsa golang.org/x/crypto/ssh golang.org/x/crypto/ssh/internal/bcrypt_pbkdf -# golang.org/x/net v0.8.0 +# golang.org/x/net v0.10.0 ## explicit; go 1.17 golang.org/x/net/bpf golang.org/x/net/context @@ -1111,7 +1111,7 @@ golang.org/x/oauth2/jwt golang.org/x/sync/errgroup golang.org/x/sync/semaphore golang.org/x/sync/syncmap -# golang.org/x/sys v0.6.0 +# golang.org/x/sys v0.8.0 ## explicit; go 1.17 golang.org/x/sys/cpu golang.org/x/sys/execabs @@ -1123,7 +1123,7 @@ golang.org/x/sys/windows/svc golang.org/x/sys/windows/svc/debug golang.org/x/sys/windows/svc/eventlog golang.org/x/sys/windows/svc/mgr -# golang.org/x/text v0.8.0 +# golang.org/x/text v0.9.0 ## explicit; go 1.17 golang.org/x/text/encoding golang.org/x/text/encoding/internal From f7298b326e6eec22ddff7af6190c5a9f641822ad Mon Sep 17 00:00:00 2001 From: Jan Garcia Date: Thu, 8 Jun 2023 22:18:59 +0200 Subject: [PATCH 078/293] vendor: github.com/sirupsen/logrus v1.9.3 Signed-off-by: Jan Garcia (cherry picked from commit 197b0b16e3de255556b3ad2a00290d81e80b0435) Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 +-- vendor/github.com/sirupsen/logrus/writer.go | 34 ++++++++++++++++++++- vendor/modules.txt | 2 +- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/vendor.mod b/vendor.mod index f21c5c2540d07..d136f01fcd153 100644 --- a/vendor.mod +++ b/vendor.mod @@ -78,7 +78,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/rootless-containers/rootlesskit v1.1.1 - github.com/sirupsen/logrus v1.9.2 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/tonistiigi/fsutil v0.0.0-20230105215944-fb433841cbfa diff --git a/vendor.sum b/vendor.sum index dbce2db84b597..010ad3565a1b2 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1302,8 +1302,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= diff --git a/vendor/github.com/sirupsen/logrus/writer.go b/vendor/github.com/sirupsen/logrus/writer.go index 72e8e3a1b65f1..074fd4b8bd78c 100644 --- a/vendor/github.com/sirupsen/logrus/writer.go +++ b/vendor/github.com/sirupsen/logrus/writer.go @@ -4,6 +4,7 @@ import ( "bufio" "io" "runtime" + "strings" ) // Writer at INFO level. See WriterLevel for details. @@ -20,15 +21,18 @@ func (logger *Logger) WriterLevel(level Level) *io.PipeWriter { return NewEntry(logger).WriterLevel(level) } +// Writer returns an io.Writer that writes to the logger at the info log level func (entry *Entry) Writer() *io.PipeWriter { return entry.WriterLevel(InfoLevel) } +// WriterLevel returns an io.Writer that writes to the logger at the given log level func (entry *Entry) WriterLevel(level Level) *io.PipeWriter { reader, writer := io.Pipe() var printFunc func(args ...interface{}) + // Determine which log function to use based on the specified log level switch level { case TraceLevel: printFunc = entry.Trace @@ -48,23 +52,51 @@ func (entry *Entry) WriterLevel(level Level) *io.PipeWriter { printFunc = entry.Print } + // Start a new goroutine to scan the input and write it to the logger using the specified print function. + // It splits the input into chunks of up to 64KB to avoid buffer overflows. go entry.writerScanner(reader, printFunc) + + // Set a finalizer function to close the writer when it is garbage collected runtime.SetFinalizer(writer, writerFinalizer) return writer } +// writerScanner scans the input from the reader and writes it to the logger func (entry *Entry) writerScanner(reader *io.PipeReader, printFunc func(args ...interface{})) { scanner := bufio.NewScanner(reader) + + // Set the buffer size to the maximum token size to avoid buffer overflows + scanner.Buffer(make([]byte, bufio.MaxScanTokenSize), bufio.MaxScanTokenSize) + + // Define a split function to split the input into chunks of up to 64KB + chunkSize := bufio.MaxScanTokenSize // 64KB + splitFunc := func(data []byte, atEOF bool) (int, []byte, error) { + if len(data) >= chunkSize { + return chunkSize, data[:chunkSize], nil + } + + return bufio.ScanLines(data, atEOF) + } + + // Use the custom split function to split the input + scanner.Split(splitFunc) + + // Scan the input and write it to the logger using the specified print function for scanner.Scan() { - printFunc(scanner.Text()) + printFunc(strings.TrimRight(scanner.Text(), "\r\n")) } + + // If there was an error while scanning the input, log an error if err := scanner.Err(); err != nil { entry.Errorf("Error while reading from Writer: %s", err) } + + // Close the reader when we are done reader.Close() } +// WriterFinalizer is a finalizer function that closes then given writer when it is garbage collected func writerFinalizer(writer *io.PipeWriter) { writer.Close() } diff --git a/vendor/modules.txt b/vendor/modules.txt index 4701379a1650b..ab048aba45842 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -896,7 +896,7 @@ github.com/secure-systems-lab/go-securesystemslib/dsse # github.com/shibumi/go-pathspec v1.3.0 ## explicit; go 1.17 github.com/shibumi/go-pathspec -# github.com/sirupsen/logrus v1.9.2 +# github.com/sirupsen/logrus v1.9.3 ## explicit; go 1.13 github.com/sirupsen/logrus # github.com/spdx/tools-golang v0.3.1-0.20230104082527-d6f58551be3f From 789a8755b8c4ea0238530441c4bf576a5a9f4982 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 9 Jun 2023 15:21:44 +0200 Subject: [PATCH 079/293] run `getent` with a noop stdin Signed-off-by: Nicolas De Loof (cherry picked from commit 3cc5d62f8a653d235bd96a450fa58fee68808c0d) Signed-off-by: Sebastiaan van Stijn --- integration/container/copy_test.go | 54 +++++++++++++++++++----------- pkg/idtools/idtools_unix.go | 5 ++- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/integration/container/copy_test.go b/integration/container/copy_test.go index 5214d15be9760..0b40182e56d56 100644 --- a/integration/container/copy_test.go +++ b/integration/container/copy_test.go @@ -65,6 +65,34 @@ func TestCopyToContainerPathDoesNotExist(t *testing.T) { func TestCopyEmptyFile(t *testing.T) { defer setupTest(t)() + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(ctx, t, apiclient) + + // empty content + dstDir, _ := makeEmptyArchive(t) + err := apiclient.CopyToContainer(ctx, cid, dstDir, bytes.NewReader([]byte("")), types.CopyToContainerOptions{}) + assert.NilError(t, err) + + // tar with empty file + dstDir, preparedArchive := makeEmptyArchive(t) + err = apiclient.CopyToContainer(ctx, cid, dstDir, preparedArchive, types.CopyToContainerOptions{}) + assert.NilError(t, err) + + // tar with empty file archive mode + dstDir, preparedArchive = makeEmptyArchive(t) + err = apiclient.CopyToContainer(ctx, cid, dstDir, preparedArchive, types.CopyToContainerOptions{ + CopyUIDGID: true, + }) + assert.NilError(t, err) + + // copy from empty file + rdr, _, err := apiclient.CopyFromContainer(ctx, cid, dstDir) + assert.NilError(t, err) + defer rdr.Close() +} + +func makeEmptyArchive(t *testing.T) (string, io.ReadCloser) { tmpDir := t.TempDir() srcPath := filepath.Join(tmpDir, "empty-file.txt") err := os.WriteFile(srcPath, []byte(""), 0400) @@ -77,30 +105,18 @@ func TestCopyEmptyFile(t *testing.T) { srcArchive, err := archive.TarResource(srcInfo) assert.NilError(t, err) - defer srcArchive.Close() + t.Cleanup(func() { + srcArchive.Close() + }) ctrPath := "/empty-file.txt" dstInfo := archive.CopyInfo{Path: ctrPath} dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo) assert.NilError(t, err) - defer preparedArchive.Close() - - ctx := context.Background() - apiclient := testEnv.APIClient() - cid := container.Create(ctx, t, apiclient) - - // empty content - err = apiclient.CopyToContainer(ctx, cid, dstDir, bytes.NewReader([]byte("")), types.CopyToContainerOptions{}) - assert.NilError(t, err) - - // tar with empty file - err = apiclient.CopyToContainer(ctx, cid, dstDir, preparedArchive, types.CopyToContainerOptions{}) - assert.NilError(t, err) - - // copy from empty file - rdr, _, err := apiclient.CopyFromContainer(ctx, cid, dstDir) - assert.NilError(t, err) - defer rdr.Close() + t.Cleanup(func() { + preparedArchive.Close() + }) + return dstDir, preparedArchive } func TestCopyToContainerPathIsNotDir(t *testing.T) { diff --git a/pkg/idtools/idtools_unix.go b/pkg/idtools/idtools_unix.go index 72e9c08a103b4..a4001c3b87456 100644 --- a/pkg/idtools/idtools_unix.go +++ b/pkg/idtools/idtools_unix.go @@ -167,7 +167,10 @@ func callGetent(database, key string) (io.Reader, error) { if getentCmd == "" { return nil, fmt.Errorf("unable to find getent command") } - out, err := exec.Command(getentCmd, database, key).CombinedOutput() + command := exec.Command(getentCmd, database, key) + // we run getent within container filesystem, but without /dev so /dev/null is not available for exec to mock stdin + command.Stdin = io.NopCloser(bytes.NewReader(nil)) + out, err := command.CombinedOutput() if err != nil { exitCode, errC := getExitCode(err) if errC != nil { From 5652c596474615a156a55ed237749508f578fde5 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 20 Jun 2023 10:03:22 +0200 Subject: [PATCH 080/293] testing: temporarily pin docker-py tests to use "bullseye" The official Python images on Docker Hub switched to debian bookworm, which is now the current stable version of Debian. However, the location of the apt repository config file changed, which causes the Dockerfile build to fail; Loaded image: emptyfs:latest Loaded image ID: sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43 INFO: Building docker-sdk-python3:5.0.3... tests/Dockerfile:6 -------------------- 5 | ARG APT_MIRROR 6 | >>> RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ 7 | >>> && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list 8 | -------------------- ERROR: failed to solve: process "/bin/sh -c sed -ri \"s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g\" /etc/apt/sources.list && sed -ri \"s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g\" /etc/apt/sources.list" did not complete successfully: exit code: 2 This needs to be fixed in docker-py, but in the meantime, we can pin to the bullseye variant. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 19d860fa9dd25f09cca979830d3a4ccaeb680529) Signed-off-by: Sebastiaan van Stijn --- hack/make/test-docker-py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hack/make/test-docker-py b/hack/make/test-docker-py index a3c09a29c21fb..2a5878d57c0b2 100644 --- a/hack/make/test-docker-py +++ b/hack/make/test-docker-py @@ -9,6 +9,12 @@ source hack/make/.integration-test-helpers #: exit status 128 : "${DOCKER_PY_COMMIT:=5.0.3}" +# The version (and variant) of the python image to use for the tests; +# see https://github.com/docker/docker-py/blob/5.0.3/tests/Dockerfile#L1C5-L3 +# +# TODO remove once https://github.com/docker/docker-py/pull/3145 is merged. +: "${PYTHON_VERSION:=3.7-bullseye}" + # custom options to pass py.test # # This option can be used to temporarily skip flaky tests (using the `--deselect` @@ -50,7 +56,7 @@ source hack/make/.integration-test-helpers [ -z "${TESTDEBUG}" ] && build_opts="--quiet" [ -f /.dockerenv ] || build_opts="${build_opts} --network=host" # shellcheck disable=SC2086 - exec docker build ${build_opts} -t "${docker_py_image}" -f tests/Dockerfile "https://github.com/docker/docker-py.git#${DOCKER_PY_COMMIT}" + exec docker build ${build_opts} --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -t "${docker_py_image}" -f tests/Dockerfile "https://github.com/docker/docker-py.git#${DOCKER_PY_COMMIT}" ) fi From aec7a80c6f0e0ff848a2370a3e0b3ece065ee823 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Sat, 17 Jun 2023 17:23:01 -0600 Subject: [PATCH 081/293] c8d: Use reference counting while mounting a snapshot Some snapshotters (like overlayfs or zfs) can't mount the same directories twice. For example if the same directroy is used as an upper directory in two mounts the kernel will output this warning: overlayfs: upperdir is in-use as upperdir/workdir of another mount, accessing files from both mounts will result in undefined behavior. And indeed accessing the files from both mounts will result in an "No such file or directory" error. This change introduces reference counts for the mounts, if a directory is already mounted the mount interface will only increment the mount counter and return the mount target effectively making sure that the filesystem doesn't end up in an undefined behavior. Signed-off-by: Djordje Lukic (cherry picked from commit 32d58144fd5ed2020a3a4629d651e69ea5c5177a) Signed-off-by: Bjorn Neergaard --- daemon/containerd/mount.go | 23 ++--- daemon/containerd/service.go | 16 ++-- daemon/daemon.go | 14 +-- daemon/daemon_unix.go | 10 +- daemon/oci_linux.go | 30 ++---- daemon/snapshotter/mount.go | 141 ++++++++++++++++++++++++++++ daemon/snapshotter/mount_default.go | 17 ++++ daemon/snapshotter/mount_windows.go | 18 ++++ daemon/start.go | 9 +- 9 files changed, 211 insertions(+), 67 deletions(-) create mode 100644 daemon/snapshotter/mount.go create mode 100644 daemon/snapshotter/mount_default.go create mode 100644 daemon/snapshotter/mount_windows.go diff --git a/daemon/containerd/mount.go b/daemon/containerd/mount.go index ed58e761dce60..6f36364f7c25d 100644 --- a/daemon/containerd/mount.go +++ b/daemon/containerd/mount.go @@ -3,9 +3,7 @@ package containerd import ( "context" "fmt" - "os" - "github.com/containerd/containerd/mount" "github.com/docker/docker/container" "github.com/sirupsen/logrus" ) @@ -19,17 +17,13 @@ func (i *ImageService) Mount(ctx context.Context, container *container.Container return err } - // The temporary location will be under /var/lib/docker/... because - // we set the `TMPDIR` - root, err := os.MkdirTemp("", fmt.Sprintf("%s_rootfs-mount", container.ID)) - if err != nil { - return fmt.Errorf("failed to create temp dir: %w", err) - } - - if err := mount.All(mounts, root); err != nil { + var root string + if root, err = i.refCountMounter.Mount(mounts, container.ID); err != nil { return fmt.Errorf("failed to mount %s: %w", root, err) } + logrus.WithField("container", container.ID).Debugf("container mounted via snapshotter: %v", root) + container.BaseFS = root return nil } @@ -38,15 +32,10 @@ func (i *ImageService) Mount(ctx context.Context, container *container.Container func (i *ImageService) Unmount(ctx context.Context, container *container.Container) error { root := container.BaseFS - if err := mount.UnmountAll(root, 0); err != nil { + if err := i.refCountMounter.Unmount(root); err != nil { + logrus.WithField("container", container.ID).WithError(err).Error("error unmounting container") return fmt.Errorf("failed to unmount %s: %w", root, err) } - if err := os.Remove(root); err != nil { - logrus.WithError(err).WithField("dir", root).Error("failed to remove mount temp dir") - } - - container.BaseFS = "" - return nil } diff --git a/daemon/containerd/service.go b/daemon/containerd/service.go index 0aa0c39146773..fce32665a845c 100644 --- a/daemon/containerd/service.go +++ b/daemon/containerd/service.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/container" daemonevents "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/images" + "github.com/docker/docker/daemon/snapshotter" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -35,6 +36,7 @@ type ImageService struct { registryService RegistryConfigProvider eventsService *daemonevents.Events pruneRunning atomic.Bool + refCountMounter snapshotter.Mounter } type RegistryHostsProvider interface { @@ -47,12 +49,13 @@ type RegistryConfigProvider interface { } type ImageServiceConfig struct { - Client *containerd.Client - Containers container.Store - Snapshotter string - HostsProvider RegistryHostsProvider - Registry RegistryConfigProvider - EventsService *daemonevents.Events + Client *containerd.Client + Containers container.Store + Snapshotter string + HostsProvider RegistryHostsProvider + Registry RegistryConfigProvider + EventsService *daemonevents.Events + RefCountMounter snapshotter.Mounter } // NewService creates a new ImageService. @@ -64,6 +67,7 @@ func NewService(config ImageServiceConfig) *ImageService { registryHosts: config.HostsProvider, registryService: config.Registry, eventsService: config.EventsService, + refCountMounter: config.RefCountMounter, } } diff --git a/daemon/daemon.go b/daemon/daemon.go index 9be2f289696af..a00a405dc18bf 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -40,6 +40,7 @@ import ( "github.com/docker/docker/daemon/images" dlogger "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/network" + "github.com/docker/docker/daemon/snapshotter" "github.com/docker/docker/daemon/stats" "github.com/docker/docker/distribution" dmetadata "github.com/docker/docker/distribution/metadata" @@ -1023,12 +1024,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, err } d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{ - Client: d.containerdCli, - Containers: d.containers, - Snapshotter: driverName, - HostsProvider: d, - Registry: d.registryService, - EventsService: d.EventsService, + Client: d.containerdCli, + Containers: d.containers, + Snapshotter: driverName, + HostsProvider: d, + Registry: d.registryService, + EventsService: d.EventsService, + RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping), }) } else { layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{ diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 4b504c871eb40..11e691a2b7769 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -1394,19 +1394,13 @@ func (daemon *Daemon) registerLinks(container *container.Container, hostConfig * // conditionalMountOnStart is a platform specific helper function during the // container start to call mount. func (daemon *Daemon) conditionalMountOnStart(container *container.Container) error { - if !daemon.UsesSnapshotter() { - return daemon.Mount(container) - } - return nil + return daemon.Mount(container) } // conditionalUnmountOnCleanup is a platform specific helper function called // during the cleanup of a container to unmount. func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) error { - if !daemon.UsesSnapshotter() { - return daemon.Unmount(container) - } - return nil + return daemon.Unmount(container) } // setDefaultIsolation determines the default isolation mode for the diff --git a/daemon/oci_linux.go b/daemon/oci_linux.go index 015a429944d77..5ad7677c84a25 100644 --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -718,21 +718,19 @@ func sysctlExists(s string) bool { // WithCommonOptions sets common docker options func WithCommonOptions(daemon *Daemon, c *container.Container) coci.SpecOpts { return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error { - if c.BaseFS == "" && !daemon.UsesSnapshotter() { + if c.BaseFS == "" { return errors.New("populateCommonSpec: BaseFS of container " + c.ID + " is unexpectedly empty") } linkedEnv, err := daemon.setupLinkedContainers(c) if err != nil { return err } - if !daemon.UsesSnapshotter() { - s.Root = &specs.Root{ - Path: c.BaseFS, - Readonly: c.HostConfig.ReadonlyRootfs, - } - if err := c.SetupWorkingDirectory(daemon.idMapping.RootPair()); err != nil { - return err - } + s.Root = &specs.Root{ + Path: c.BaseFS, + Readonly: c.HostConfig.ReadonlyRootfs, + } + if err := c.SetupWorkingDirectory(daemon.idMapping.RootPair()); err != nil { + return err } cwd := c.Config.WorkingDir if len(cwd) == 0 { @@ -1023,20 +1021,8 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r WithSelinux(c), WithOOMScore(&c.HostConfig.OomScoreAdj), coci.WithAnnotations(c.HostConfig.Annotations), + WithUser(c), ) - if daemon.UsesSnapshotter() { - s.Root = &specs.Root{ - Path: "rootfs", - } - if c.Config.User != "" { - opts = append(opts, coci.WithUser(c.Config.User)) - } - if c.Config.WorkingDir != "" { - opts = append(opts, coci.WithProcessCwd(c.Config.WorkingDir)) - } - } else { - opts = append(opts, WithUser(c)) - } if c.NoNewPrivileges { opts = append(opts, coci.WithNoNewPrivileges) diff --git a/daemon/snapshotter/mount.go b/daemon/snapshotter/mount.go new file mode 100644 index 0000000000000..0dc3f0bdd2cab --- /dev/null +++ b/daemon/snapshotter/mount.go @@ -0,0 +1,141 @@ +package snapshotter + +import ( + "os" + "path/filepath" + + "github.com/containerd/containerd/mount" + "github.com/docker/docker/daemon/graphdriver" + "github.com/docker/docker/pkg/idtools" + "github.com/moby/locker" + "github.com/sirupsen/logrus" +) + +const mountsDir = "rootfs" + +// List of known filesystems that can't be re-mounted or have shared layers +var refCountedFileSystems = []string{"overlayfs", "zfs", "fuse-overlayfs"} + +// Mounter handles mounting/unmounting things coming in from a snapshotter +// with optional reference counting if needed by the filesystem +type Mounter interface { + // Mount mounts the rootfs for a container and returns the mount point + Mount(mounts []mount.Mount, containerID string) (string, error) + // Unmount unmounts the container rootfs + Unmount(target string) error +} + +// inSlice tests whether a string is contained in a slice of strings or not. +// Comparison is case sensitive +func inSlice(slice []string, s string) bool { + for _, ss := range slice { + if s == ss { + return true + } + } + return false +} + +// NewMounter creates a new mounter for the provided snapshotter +func NewMounter(home string, snapshotter string, idMap idtools.IdentityMapping) Mounter { + if inSlice(refCountedFileSystems, snapshotter) { + return &refCountMounter{ + home: home, + snapshotter: snapshotter, + rc: graphdriver.NewRefCounter(checker()), + locker: locker.New(), + idMap: idMap, + } + } + + return mounter{ + home: home, + snapshotter: snapshotter, + idMap: idMap, + } +} + +type refCountMounter struct { + home string + snapshotter string + rc *graphdriver.RefCounter + locker *locker.Locker + idMap idtools.IdentityMapping +} + +func (m *refCountMounter) Mount(mounts []mount.Mount, containerID string) (target string, retErr error) { + target = filepath.Join(m.home, mountsDir, m.snapshotter, containerID) + + _, err := os.Stat(target) + if err != nil && !os.IsNotExist(err) { + return "", err + } + + if count := m.rc.Increment(target); count > 1 { + return target, nil + } + + m.locker.Lock(target) + defer m.locker.Unlock(target) + + defer func() { + if retErr != nil { + if c := m.rc.Decrement(target); c <= 0 { + if mntErr := unmount(target); mntErr != nil { + logrus.Errorf("error unmounting %s: %v", target, mntErr) + } + if rmErr := os.Remove(target); rmErr != nil && !os.IsNotExist(rmErr) { + logrus.Debugf("Failed to remove %s: %v: %v", target, rmErr, err) + } + } + } + }() + + root := m.idMap.RootPair() + if err := idtools.MkdirAllAndChown(target, 0700, root); err != nil { + return "", err + } + + return target, mount.All(mounts, target) +} + +func (m *refCountMounter) Unmount(target string) error { + if count := m.rc.Decrement(target); count > 0 { + return nil + } + + m.locker.Lock(target) + defer m.locker.Unlock(target) + + if err := unmount(target); err != nil { + logrus.Debugf("Failed to unmount %s: %v", target, err) + } + + if err := os.Remove(target); err != nil { + logrus.WithError(err).WithField("dir", target).Error("failed to remove mount temp dir") + } + + return nil +} + +type mounter struct { + home string + snapshotter string + idMap idtools.IdentityMapping +} + +func (m mounter) Mount(mounts []mount.Mount, containerID string) (string, error) { + target := filepath.Join(m.home, mountsDir, m.snapshotter, containerID) + + root := m.idMap.RootPair() + if err := idtools.MkdirAndChown(target, 0700, root); err != nil { + return "", err + } + + return target, mount.All(mounts, target) +} + +func (m mounter) Unmount(target string) error { + return unmount(target) + +} diff --git a/daemon/snapshotter/mount_default.go b/daemon/snapshotter/mount_default.go new file mode 100644 index 0000000000000..8203a9c47936b --- /dev/null +++ b/daemon/snapshotter/mount_default.go @@ -0,0 +1,17 @@ +//go:build !windows + +package snapshotter + +import ( + "github.com/containerd/containerd/mount" + "github.com/docker/docker/daemon/graphdriver" + "golang.org/x/sys/unix" +) + +func checker() graphdriver.Checker { + return graphdriver.NewDefaultChecker() +} + +func unmount(target string) error { + return mount.Unmount(target, unix.MNT_DETACH) +} diff --git a/daemon/snapshotter/mount_windows.go b/daemon/snapshotter/mount_windows.go new file mode 100644 index 0000000000000..f43cfe24c46f7 --- /dev/null +++ b/daemon/snapshotter/mount_windows.go @@ -0,0 +1,18 @@ +package snapshotter + +import "github.com/containerd/containerd/mount" + +type winChecker struct { +} + +func (c *winChecker) IsMounted(path string) bool { + return false +} + +func checker() *winChecker { + return &winChecker{} +} + +func unmount(target string) error { + return mount.Unmount(target, 0) +} diff --git a/daemon/start.go b/daemon/start.go index 0b4eb6d67bcce..2e0b9e6be847d 100644 --- a/daemon/start.go +++ b/daemon/start.go @@ -5,7 +5,6 @@ import ( "runtime" "time" - "github.com/containerd/containerd" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" @@ -178,13 +177,7 @@ func (daemon *Daemon) containerStart(ctx context.Context, container *container.C return err } - newContainerOpts := []containerd.NewContainerOpts{} - if daemon.UsesSnapshotter() { - newContainerOpts = append(newContainerOpts, containerd.WithSnapshotter(container.Driver)) - newContainerOpts = append(newContainerOpts, containerd.WithSnapshot(container.ID)) - } - - ctr, err := libcontainerd.ReplaceContainer(ctx, daemon.containerd, container.ID, spec, shim, createOptions, newContainerOpts...) + ctr, err := libcontainerd.ReplaceContainer(ctx, daemon.containerd, container.ID, spec, shim, createOptions) if err != nil { return setExitCodeFromError(container.SetExitCode, err) } From 7db3243e34f2804797d48d3189eace7491879a7a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 13 Jun 2023 23:18:04 +0200 Subject: [PATCH 082/293] don't cancel container stop when cancelling context Commit 90de570cfa5f6de0b02bdf2e794627fbf5b0dfc8 passed through the request context to daemon.ContainerStop(). As a result, cancelling the context would cancel the "graceful" stop of the container, and would proceed with forcefully killing the container. This patch partially reverts the changes from 90de570cfa5f6de0b02bdf2e794627fbf5b0dfc8 and breaks the context to prevent cancelling the context from cancelling the stop. Signed-off-by: Sebastiaan van Stijn Signed-off-by: Sebastiaan van Stijn (cherry picked from commit fc94ed0a86dbac35f9e4cbdd778ead7bc2d53bf9) Signed-off-by: Sebastiaan van Stijn --- daemon/stop.go | 8 ++- integration/container/stop_linux_test.go | 80 ++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/daemon/stop.go b/daemon/stop.go index 5a659c1cc8b21..0d29e5c99608a 100644 --- a/daemon/stop.go +++ b/daemon/stop.go @@ -36,7 +36,13 @@ func (daemon *Daemon) ContainerStop(ctx context.Context, name string, options co } // containerStop sends a stop signal, waits, sends a kill signal. -func (daemon *Daemon) containerStop(ctx context.Context, ctr *container.Container, options containertypes.StopOptions) (retErr error) { +func (daemon *Daemon) containerStop(_ context.Context, ctr *container.Container, options containertypes.StopOptions) (retErr error) { + // Deliberately using a local context here, because cancelling the + // request should not cancel the stop. + // + // TODO(thaJeztah): pass context, and use context.WithoutCancel() once available: https://github.com/golang/go/issues/40221 + ctx := context.Background() + if !ctr.IsRunning() { return nil } diff --git a/integration/container/stop_linux_test.go b/integration/container/stop_linux_test.go index 0535ce778777d..9cfe42351d7e7 100644 --- a/integration/container/stop_linux_test.go +++ b/integration/container/stop_linux_test.go @@ -1,8 +1,10 @@ package container // import "github.com/docker/docker/integration/container" import ( + "bytes" "context" "fmt" + "io" "strconv" "strings" "testing" @@ -10,8 +12,12 @@ import ( "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" + "github.com/docker/docker/errdefs" "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/pkg/stdcopy" "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" "gotest.tools/v3/icmd" "gotest.tools/v3/poll" "gotest.tools/v3/skip" @@ -98,3 +104,77 @@ func TestDeleteDevicemapper(t *testing.T) { err = client.ContainerRemove(ctx, id, types.ContainerRemoveOptions{}) assert.NilError(t, err) } + +// TestStopContainerWithTimeoutCancel checks that ContainerStop is not cancelled +// if the request is cancelled. +// See issue https://github.com/moby/moby/issues/45731 +func TestStopContainerWithTimeoutCancel(t *testing.T) { + t.Parallel() + defer setupTest(t)() + apiClient := testEnv.APIClient() + t.Cleanup(func() { _ = apiClient.Close() }) + + ctx := context.Background() + id := container.Run(ctx, t, apiClient, + container.WithCmd("sh", "-c", "trap 'echo received TERM' TERM; while true; do usleep 10; done"), + ) + poll.WaitOn(t, container.IsInState(ctx, apiClient, id, "running")) + + ctxCancel, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + const stopTimeout = 3 + + stoppedCh := make(chan error) + go func() { + sto := stopTimeout + stoppedCh <- apiClient.ContainerStop(ctxCancel, id, containertypes.StopOptions{Timeout: &sto}) + }() + + poll.WaitOn(t, logsContains(ctx, apiClient, id, "received TERM")) + + // Cancel the context once we verified the container was signaled, and check + // that the container is not killed immediately + cancel() + + select { + case stoppedErr := <-stoppedCh: + assert.Check(t, is.ErrorType(stoppedErr, errdefs.IsCancelled)) + case <-time.After(5 * time.Second): + t.Fatal("timeout waiting for stop request to be cancelled") + } + inspect, err := apiClient.ContainerInspect(ctx, id) + assert.Check(t, err) + assert.Check(t, inspect.State.Running) + + // container should be stopped after stopTimeout is reached. The daemon.containerStop + // code is rather convoluted, and waits another 2 seconds for the container to + // terminate after signaling it; + // https://github.com/moby/moby/blob/97455cc31ffa08078db6591f018256ed59c35bbc/daemon/stop.go#L101-L112 + // + // Adding 3 seconds to the specified stopTimeout to take this into account, + // and add another second margin to try to avoid flakiness. + poll.WaitOn(t, container.IsStopped(ctx, apiClient, id), poll.WithTimeout((3+stopTimeout)*time.Second)) +} + +// logsContains verifies the container contains the given text in the log's stdout. +func logsContains(ctx context.Context, client client.APIClient, containerID string, logString string) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + logs, err := client.ContainerLogs(ctx, containerID, types.ContainerLogsOptions{ + ShowStdout: true, + }) + if err != nil { + return poll.Error(err) + } + defer logs.Close() + + var stdout bytes.Buffer + _, err = stdcopy.StdCopy(&stdout, io.Discard, logs) + if err != nil { + return poll.Error(err) + } + if strings.Contains(stdout.String(), logString) { + return poll.Success() + } + return poll.Continue("waiting for logstring '%s' in container", logString) + } +} From 67762798961e866ecae5e6df4de8841ced49a000 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 13 Jun 2023 13:33:33 +0200 Subject: [PATCH 083/293] daemon: registerName(): don't reserve name twice daemon.generateNewName() already reserves the generated name, but its name did not indicate it did. The daemon.registerName() assumed that the generated name still had to be reserved, which could mean it would try to reserve the same name again. This patch renames daemon.generateNewName to daemon.generateAndReserveName to make it clearer what it does, and updates registerName() to return early if it successfully generated (and registered) the container name. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 3ba67ee214927b681a560ba4d2a4ac7f3a993ad9) Signed-off-by: Sebastiaan van Stijn --- daemon/names.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/daemon/names.go b/daemon/names.go index b6d6e4da849c1..ccb6467ba368a 100644 --- a/daemon/names.go +++ b/daemon/names.go @@ -26,11 +26,12 @@ func (daemon *Daemon) registerName(container *container.Container) error { return err } if container.Name == "" { - name, err := daemon.generateNewName(container.ID) + name, err := daemon.generateAndReserveName(container.ID) if err != nil { return err } container.Name = name + return nil } return daemon.containersReplica.ReserveName(container.Name, container.ID) } @@ -42,7 +43,7 @@ func (daemon *Daemon) generateIDAndName(name string) (string, string, error) { ) if name == "" { - if name, err = daemon.generateNewName(id); err != nil { + if name, err = daemon.generateAndReserveName(id); err != nil { return "", "", err } return id, name, nil @@ -81,7 +82,7 @@ func (daemon *Daemon) releaseName(name string) { daemon.containersReplica.ReleaseName(name) } -func (daemon *Daemon) generateNewName(id string) (string, error) { +func (daemon *Daemon) generateAndReserveName(id string) (string, error) { var name string for i := 0; i < 6; i++ { name = namesgenerator.GetRandomName(i) From 5e48bbd14c49e4605f1d020e25add1a8e034e529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 21 Jun 2023 14:15:05 +0200 Subject: [PATCH 084/293] contrib/busybox: Update to FRP-5007-g82accfc19 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit e01022318621102f6692ddc37a5b0c475dfe5ab4) Signed-off-by: Paweł Gronowski --- contrib/busybox/Dockerfile | 4 ++-- integration-cli/docker_api_build_test.go | 11 ++--------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/contrib/busybox/Dockerfile b/contrib/busybox/Dockerfile index 51a27e0b7338a..ad3068428d607 100644 --- a/contrib/busybox/Dockerfile +++ b/contrib/busybox/Dockerfile @@ -10,10 +10,10 @@ # To publish: Needs someone with publishing rights ARG WINDOWS_BASE_IMAGE=mcr.microsoft.com/windows/servercore ARG WINDOWS_BASE_IMAGE_TAG=ltsc2022 -ARG BUSYBOX_VERSION=FRP-3329-gcf0fa4d13 +ARG BUSYBOX_VERSION=FRP-5007-g82accfc19 # Checksum taken from https://frippery.org/files/busybox/SHA256SUM -ARG BUSYBOX_SHA256SUM=bfaeb88638e580fc522a68e69072e305308f9747563e51fa085eec60ca39a5ae +ARG BUSYBOX_SHA256SUM=2d6fff0b2de5c034c92990d696c0d85a677b8a75931fa1ec30694fbf1f1df5c9 FROM ${WINDOWS_BASE_IMAGE}:${WINDOWS_BASE_IMAGE_TAG} RUN mkdir C:\tmp && mkdir C:\bin diff --git a/integration-cli/docker_api_build_test.go b/integration-cli/docker_api_build_test.go index 4b1008ec87fd0..2bc9c9dd98ae4 100644 --- a/integration-cli/docker_api_build_test.go +++ b/integration-cli/docker_api_build_test.go @@ -24,17 +24,10 @@ import ( func (s *DockerAPISuite) TestBuildAPIDockerFileRemote(c *testing.T) { testRequires(c, NotUserNamespace) - var testD string - if testEnv.OSType == "windows" { - testD = `FROM busybox -RUN find / -name ba* -RUN find /tmp/` - } else { - // -xdev is required because sysfs can cause EPERM - testD = `FROM busybox + // -xdev is required because sysfs can cause EPERM + testD := `FROM busybox RUN find / -xdev -name ba* RUN find /tmp/` - } server := fakestorage.New(c, "", fakecontext.WithFiles(map[string]string{"testD": testD})) defer server.Close() From c92fd5220a45dfea521e4354ecb092d02e79d1d5 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Tue, 20 Jun 2023 12:59:08 -0600 Subject: [PATCH 085/293] c8d: mark stargz as requiring reference-counted mounts The stargz snapshotter cannot be re-mounted, so the reference-counted path must be used. Co-authored-by: Djordje Lukic Signed-off-by: Bjorn Neergaard (cherry picked from commit 21c0a54a6b2dc059fae62f46a23d338a35fe6d1d) Signed-off-by: Bjorn Neergaard --- daemon/snapshotter/mount.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/snapshotter/mount.go b/daemon/snapshotter/mount.go index 0dc3f0bdd2cab..f156bb2ab27dd 100644 --- a/daemon/snapshotter/mount.go +++ b/daemon/snapshotter/mount.go @@ -14,7 +14,7 @@ import ( const mountsDir = "rootfs" // List of known filesystems that can't be re-mounted or have shared layers -var refCountedFileSystems = []string{"overlayfs", "zfs", "fuse-overlayfs"} +var refCountedFileSystems = []string{"fuse-overlayfs", "overlayfs", "stargz", "zfs"} // Mounter handles mounting/unmounting things coming in from a snapshotter // with optional reference counting if needed by the filesystem From 0a6a5a9140e8381f528b1c1861aec971a247081b Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Mon, 5 Jun 2023 18:30:30 -0400 Subject: [PATCH 086/293] daemon: modernize oci_linux_test.go Switch to using t.TempDir() instead of rolling our own. Clean up mounts leaked by the tests as otherwise the tests fail due to the leaked mounts because unlike the old cleanup code, t.TempDir() cleanup does not ignore errors from os.RemoveAll. Signed-off-by: Cory Snider (cherry picked from commit 9ff169ccf421c00f3106481e43c5d86f77403b06) Signed-off-by: Sebastiaan van Stijn --- daemon/oci_linux_test.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/daemon/oci_linux_test.go b/daemon/oci_linux_test.go index e08f557674291..7bafd7452eb54 100644 --- a/daemon/oci_linux_test.go +++ b/daemon/oci_linux_test.go @@ -11,17 +11,18 @@ import ( "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/network" "github.com/docker/docker/libnetwork" + "golang.org/x/sys/unix" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" "gotest.tools/v3/skip" ) func setupFakeDaemon(t *testing.T, c *container.Container) *Daemon { - root, err := os.MkdirTemp("", "oci_linux_test-root") - assert.NilError(t, err) + t.Helper() + root := t.TempDir() rootfs := filepath.Join(root, "rootfs") - err = os.MkdirAll(rootfs, 0755) + err := os.MkdirAll(rootfs, 0755) assert.NilError(t, err) netController, err := libnetwork.New() @@ -49,6 +50,18 @@ func setupFakeDaemon(t *testing.T, c *container.Container) *Daemon { c.NetworkSettings = &network.Settings{Networks: make(map[string]*network.EndpointSettings)} } + // HORRIBLE HACK: clean up shm mounts leaked by some tests. Otherwise the + // offending tests would fail due to the mounts blocking the temporary + // directory from being cleaned up. + t.Cleanup(func() { + if c.ShmPath != "" { + var err error + for err == nil { // Some tests over-mount over the same path multiple times. + err = unix.Unmount(c.ShmPath, unix.MNT_DETACH) + } + } + }) + return d } @@ -60,10 +73,6 @@ func (i *fakeImageService) StorageDriver() string { return "overlay" } -func cleanupFakeContainer(c *container.Container) { - _ = os.RemoveAll(c.Root) -} - // TestTmpfsDevShmNoDupMount checks that a user-specified /dev/shm tmpfs // mount (as in "docker run --tmpfs /dev/shm:rw,size=NNN") does not result // in "Duplicate mount point" error from the engine. @@ -81,7 +90,6 @@ func TestTmpfsDevShmNoDupMount(t *testing.T) { }, } d := setupFakeDaemon(t, c) - defer cleanupFakeContainer(c) _, err := d.createSpec(context.TODO(), c) assert.Check(t, err) @@ -100,7 +108,6 @@ func TestIpcPrivateVsReadonly(t *testing.T) { }, } d := setupFakeDaemon(t, c) - defer cleanupFakeContainer(c) s, err := d.createSpec(context.TODO(), c) assert.Check(t, err) @@ -129,7 +136,6 @@ func TestSysctlOverride(t *testing.T) { }, } d := setupFakeDaemon(t, c) - defer cleanupFakeContainer(c) // Ensure that the implicit sysctl is set correctly. s, err := d.createSpec(context.TODO(), c) @@ -181,7 +187,6 @@ func TestSysctlOverrideHost(t *testing.T) { }, } d := setupFakeDaemon(t, c) - defer cleanupFakeContainer(c) // Ensure that the implicit sysctl is not set s, err := d.createSpec(context.TODO(), c) From f50cb0c7bdb65dfc263f7a4e8102b327f20d1a90 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Mon, 5 Jun 2023 18:44:51 -0400 Subject: [PATCH 087/293] daemon: stop setting container resources to zero Many of the fields in LinuxResources struct are pointers to scalars for some reason, presumably to differentiate between set-to-zero and unset when unmarshaling from JSON, despite zero being outside the acceptable range for the corresponding kernel tunables. When creating the OCI spec for a container, the daemon sets the container's OCI spec CPUShares and BlkioWeight parameters to zero when the corresponding Docker container configuration values are zero, signifying unset, despite the minimum acceptable value for CPUShares being two, and BlkioWeight ten. This has gone unnoticed as runC does not distingiush set-to-zero from unset as it also uses zero internally to represent unset for those fields. However, kata-containers v3.2.0-alpha.3 tries to apply the explicit-zero resource parameters to the container, exactly as instructed, and fails loudly. The OCI runtime-spec is silent on how the runtime should handle the case when those parameters are explicitly set to out-of-range values and kata's behaviour is not unreasonable, so the daemon must therefore be in the wrong. Translate unset values in the Docker container's resources HostConfig to omit the corresponding fields in the container's OCI spec when starting and updating a container in order to maximize compatibility with runtimes. Signed-off-by: Cory Snider (cherry picked from commit dea870f4eafac1efb1d3a70d8196da11bfa6c59c) Signed-off-by: Sebastiaan van Stijn --- daemon/daemon_unix.go | 12 +++++-- daemon/oci_linux.go | 6 ++-- daemon/oci_linux_test.go | 37 +++++++++++++++++++++ daemon/update_linux.go | 48 ++++++++++++++++++++-------- daemon/update_linux_test.go | 11 +++++++ libcontainerd/remote/client_linux.go | 4 +-- libcontainerd/types/types_linux.go | 2 +- 7 files changed, 97 insertions(+), 23 deletions(-) create mode 100644 daemon/update_linux_test.go diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 11e691a2b7769..065dee9e0d49c 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -105,7 +105,10 @@ func getMemoryResources(config containertypes.Resources) *specs.LinuxMemory { memory.KernelTCP = &config.KernelMemoryTCP } - return &memory + if memory != (specs.LinuxMemory{}) { + return &memory + } + return nil } func getPidsLimit(config containertypes.Resources) *specs.LinuxPids { @@ -127,7 +130,7 @@ func getCPUResources(config containertypes.Resources) (*specs.LinuxCPU, error) { if config.CPUShares < 0 { return nil, fmt.Errorf("shares: invalid argument") } - if config.CPUShares >= 0 { + if config.CPUShares > 0 { shares := uint64(config.CPUShares) cpu.Shares = &shares } @@ -168,7 +171,10 @@ func getCPUResources(config containertypes.Resources) (*specs.LinuxCPU, error) { cpu.RealtimeRuntime = &c } - return &cpu, nil + if cpu != (specs.LinuxCPU{}) { + return &cpu, nil + } + return nil, nil } func getBlkioWeightDevices(config containertypes.Resources) ([]specs.LinuxWeightDevice, error) { diff --git a/daemon/oci_linux.go b/daemon/oci_linux.go index 5ad7677c84a25..5c607a3359a69 100644 --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -954,13 +954,11 @@ func WithResources(c *container.Container) coci.SpecOpts { if err != nil { return err } - blkioWeight := r.BlkioWeight specResources := &specs.LinuxResources{ Memory: memoryRes, CPU: cpuRes, BlockIO: &specs.LinuxBlockIO{ - Weight: &blkioWeight, WeightDevice: weightDevices, ThrottleReadBpsDevice: readBpsDevice, ThrottleWriteBpsDevice: writeBpsDevice, @@ -969,6 +967,10 @@ func WithResources(c *container.Container) coci.SpecOpts { }, Pids: getPidsLimit(r), } + if r.BlkioWeight != 0 { + w := r.BlkioWeight + specResources.BlockIO.Weight = &w + } if s.Linux.Resources != nil && len(s.Linux.Resources.Devices) > 0 { specResources.Devices = s.Linux.Resources.Devices diff --git a/daemon/oci_linux_test.go b/daemon/oci_linux_test.go index 7bafd7452eb54..a0a0aa52604bd 100644 --- a/daemon/oci_linux_test.go +++ b/daemon/oci_linux_test.go @@ -11,6 +11,8 @@ import ( "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/network" "github.com/docker/docker/libnetwork" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -214,3 +216,38 @@ func TestGetSourceMount(t *testing.T) { _, _, err = getSourceMount(cwd) assert.NilError(t, err) } + +func TestDefaultResources(t *testing.T) { + skip.If(t, os.Getuid() != 0, "skipping test that requires root") // TODO: is this actually true? I'm guilty of following the cargo cult here. + + c := &container.Container{ + HostConfig: &containertypes.HostConfig{ + IpcMode: containertypes.IPCModeNone, + }, + } + d := setupFakeDaemon(t, c) + + s, err := d.createSpec(context.Background(), c) + assert.NilError(t, err) + checkResourcesAreUnset(t, s.Linux.Resources) +} + +func checkResourcesAreUnset(t *testing.T, r *specs.LinuxResources) { + t.Helper() + + if r != nil { + if r.Memory != nil { + assert.Check(t, is.DeepEqual(r.Memory, &specs.LinuxMemory{})) + } + if r.CPU != nil { + assert.Check(t, is.DeepEqual(r.CPU, &specs.LinuxCPU{})) + } + assert.Check(t, is.Nil(r.Pids)) + if r.BlockIO != nil { + assert.Check(t, is.DeepEqual(r.BlockIO, &specs.LinuxBlockIO{}, cmpopts.EquateEmpty())) + } + if r.Network != nil { + assert.Check(t, is.DeepEqual(r.Network, &specs.LinuxNetwork{}, cmpopts.EquateEmpty())) + } + } +} diff --git a/daemon/update_linux.go b/daemon/update_linux.go index c1d3684868db1..3105402e3ce06 100644 --- a/daemon/update_linux.go +++ b/daemon/update_linux.go @@ -11,15 +11,19 @@ import ( func toContainerdResources(resources container.Resources) *libcontainerdtypes.Resources { var r libcontainerdtypes.Resources - r.BlockIO = &specs.LinuxBlockIO{ - Weight: &resources.BlkioWeight, + if resources.BlkioWeight != 0 { + r.BlockIO = &specs.LinuxBlockIO{ + Weight: &resources.BlkioWeight, + } } - shares := uint64(resources.CPUShares) - r.CPU = &specs.LinuxCPU{ - Shares: &shares, - Cpus: resources.CpusetCpus, - Mems: resources.CpusetMems, + cpu := specs.LinuxCPU{ + Cpus: resources.CpusetCpus, + Mems: resources.CpusetMems, + } + if resources.CPUShares != 0 { + shares := uint64(resources.CPUShares) + cpu.Shares = &shares } var ( @@ -37,17 +41,33 @@ func toContainerdResources(resources container.Resources) *libcontainerdtypes.Re period = uint64(resources.CPUPeriod) } - r.CPU.Period = &period - r.CPU.Quota = "a + if period != 0 { + cpu.Period = &period + } + if quota != 0 { + cpu.Quota = "a + } - r.Memory = &specs.LinuxMemory{ - Limit: &resources.Memory, - Reservation: &resources.MemoryReservation, - Kernel: &resources.KernelMemory, + if cpu != (specs.LinuxCPU{}) { + r.CPU = &cpu } + var memory specs.LinuxMemory + if resources.Memory != 0 { + memory.Limit = &resources.Memory + } + if resources.MemoryReservation != 0 { + memory.Reservation = &resources.MemoryReservation + } + if resources.KernelMemory != 0 { + memory.Kernel = &resources.KernelMemory + } if resources.MemorySwap > 0 { - r.Memory.Swap = &resources.MemorySwap + memory.Swap = &resources.MemorySwap + } + + if memory != (specs.LinuxMemory{}) { + r.Memory = &memory } r.Pids = getPidsLimit(resources) diff --git a/daemon/update_linux_test.go b/daemon/update_linux_test.go new file mode 100644 index 0000000000000..4817c1eab7fd3 --- /dev/null +++ b/daemon/update_linux_test.go @@ -0,0 +1,11 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "testing" + + "github.com/docker/docker/api/types/container" +) + +func TestToContainerdResources_Defaults(t *testing.T) { + checkResourcesAreUnset(t, toContainerdResources(container.Resources{})) +} diff --git a/libcontainerd/remote/client_linux.go b/libcontainerd/remote/client_linux.go index dd7aee8fe85c1..3b7ee1ab6ef2a 100644 --- a/libcontainerd/remote/client_linux.go +++ b/libcontainerd/remote/client_linux.go @@ -21,9 +21,7 @@ func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) { } func (t *task) UpdateResources(ctx context.Context, resources *libcontainerdtypes.Resources) error { - // go doesn't like the alias in 1.8, this means this need to be - // platform specific - return t.Update(ctx, containerd.WithResources((*specs.LinuxResources)(resources))) + return t.Update(ctx, containerd.WithResources(resources)) } func hostIDFromMap(id uint32, mp []specs.LinuxIDMapping) int { diff --git a/libcontainerd/types/types_linux.go b/libcontainerd/types/types_linux.go index 34360ed5c2885..c91bcb9223f8c 100644 --- a/libcontainerd/types/types_linux.go +++ b/libcontainerd/types/types_linux.go @@ -27,7 +27,7 @@ func InterfaceToStats(read time.Time, v interface{}) *Stats { } // Resources defines updatable container resource values. TODO: it must match containerd upcoming API -type Resources specs.LinuxResources +type Resources = specs.LinuxResources // Checkpoints contains the details of a checkpoint type Checkpoints struct{} From 210c4d6f4b290368d3dbd8499aaf15218735620e Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Tue, 6 Jun 2023 12:57:38 -0400 Subject: [PATCH 088/293] daemon: ensure OCI options play nicely together Audit the OCI spec options used for Linux containers to ensure they are less order-dependent. Ensure they don't assume that any pointer fields are non-nil and that they don't unintentionally clobber mutations to the spec applied by other options. Signed-off-by: Cory Snider (cherry picked from commit 8a094fe60913fed77deb8275207368617f16328f) Signed-off-by: Sebastiaan van Stijn --- daemon/oci_linux.go | 90 +++++++++++++++++++------ daemon/oci_opts.go | 3 + daemon/oci_utils.go | 7 +- daemon/seccomp_linux.go | 4 ++ oci/namespaces.go | 3 + oci/oci.go | 3 + pkg/rootless/specconv/specconv_linux.go | 31 +++++---- 7 files changed, 108 insertions(+), 33 deletions(-) diff --git a/daemon/oci_linux.go b/daemon/oci_linux.go index 5c607a3359a69..64a7d3bdf2b9e 100644 --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -53,6 +53,9 @@ func WithRlimits(daemon *Daemon, c *container.Container) coci.SpecOpts { }) } + if s.Process == nil { + s.Process = &specs.Process{} + } s.Process.Rlimits = rlimits return nil } @@ -113,6 +116,9 @@ func WithRootless(daemon *Daemon) coci.SpecOpts { // WithOOMScore sets the oom score func WithOOMScore(score *int) coci.SpecOpts { return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error { + if s.Process == nil { + s.Process = &specs.Process{} + } s.Process.OOMScoreAdj = score return nil } @@ -121,6 +127,12 @@ func WithOOMScore(score *int) coci.SpecOpts { // WithSelinux sets the selinux labels func WithSelinux(c *container.Container) coci.SpecOpts { return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error { + if s.Process == nil { + s.Process = &specs.Process{} + } + if s.Linux == nil { + s.Linux = &specs.Linux{} + } s.Process.SelinuxLabel = c.GetProcessLabel() s.Linux.MountLabel = c.MountLabel return nil @@ -151,6 +163,9 @@ func WithApparmor(c *container.Container) coci.SpecOpts { return err } } + if s.Process == nil { + s.Process = &specs.Process{} + } s.Process.ApparmorProfile = appArmorProfile } return nil @@ -213,6 +228,10 @@ func getUser(c *container.Container, username string) (specs.User, error) { } func setNamespace(s *specs.Spec, ns specs.LinuxNamespace) { + if s.Linux == nil { + s.Linux = &specs.Linux{} + } + for i, n := range s.Linux.Namespaces { if n.Type == ns.Type { s.Linux.Namespaces[i] = ns @@ -606,6 +625,9 @@ func WithMounts(daemon *Daemon, c *container.Container) coci.SpecOpts { } rootpg := mountPropagationMap[s.Linux.RootfsPropagation] if rootpg != mount.SHARED && rootpg != mount.RSHARED { + if s.Linux == nil { + s.Linux = &specs.Linux{} + } s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.SHARED] } case mount.SLAVE, mount.RSLAVE: @@ -634,6 +656,9 @@ func WithMounts(daemon *Daemon, c *container.Container) coci.SpecOpts { if !fallback { rootpg := mountPropagationMap[s.Linux.RootfsPropagation] if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE { + if s.Linux == nil { + s.Linux = &specs.Linux{} + } s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE] } } @@ -689,8 +714,10 @@ func WithMounts(daemon *Daemon, c *container.Container) coci.SpecOpts { clearReadOnly(&s.Mounts[i]) } } - s.Linux.ReadonlyPaths = nil - s.Linux.MaskedPaths = nil + if s.Linux != nil { + s.Linux.ReadonlyPaths = nil + s.Linux.MaskedPaths = nil + } } // TODO: until a kernel/mount solution exists for handling remount in a user namespace, @@ -736,6 +763,9 @@ func WithCommonOptions(daemon *Daemon, c *container.Container) coci.SpecOpts { if len(cwd) == 0 { cwd = "/" } + if s.Process == nil { + s.Process = &specs.Process{} + } s.Process.Args = append([]string{c.Path}, c.Args...) // only add the custom init if it is specified and the container is running in its @@ -812,6 +842,9 @@ func WithCgroups(daemon *Daemon, c *container.Container) coci.SpecOpts { } else { cgroupsPath = filepath.Join(parent, c.ID) } + if s.Linux == nil { + s.Linux = &specs.Linux{} + } s.Linux.CgroupsPath = cgroupsPath // the rest is only needed for CPU RT controller @@ -912,8 +945,14 @@ func WithDevices(daemon *Daemon, c *container.Container) coci.SpecOpts { } } + if s.Linux == nil { + s.Linux = &specs.Linux{} + } + if s.Linux.Resources == nil { + s.Linux.Resources = &specs.LinuxResources{} + } s.Linux.Devices = append(s.Linux.Devices, devs...) - s.Linux.Resources.Devices = devPermissions + s.Linux.Resources.Devices = append(s.Linux.Resources.Devices, devPermissions...) for _, req := range c.HostConfig.DeviceRequests { if err := daemon.handleDevice(req, s); err != nil { @@ -955,28 +994,27 @@ func WithResources(c *container.Container) coci.SpecOpts { return err } - specResources := &specs.LinuxResources{ - Memory: memoryRes, - CPU: cpuRes, - BlockIO: &specs.LinuxBlockIO{ - WeightDevice: weightDevices, - ThrottleReadBpsDevice: readBpsDevice, - ThrottleWriteBpsDevice: writeBpsDevice, - ThrottleReadIOPSDevice: readIOpsDevice, - ThrottleWriteIOPSDevice: writeIOpsDevice, - }, - Pids: getPidsLimit(r), + if s.Linux == nil { + s.Linux = &specs.Linux{} + } + if s.Linux.Resources == nil { + s.Linux.Resources = &specs.LinuxResources{} + } + s.Linux.Resources.Memory = memoryRes + s.Linux.Resources.CPU = cpuRes + s.Linux.Resources.BlockIO = &specs.LinuxBlockIO{ + WeightDevice: weightDevices, + ThrottleReadBpsDevice: readBpsDevice, + ThrottleWriteBpsDevice: writeBpsDevice, + ThrottleReadIOPSDevice: readIOpsDevice, + ThrottleWriteIOPSDevice: writeIOpsDevice, } if r.BlkioWeight != 0 { w := r.BlkioWeight - specResources.BlockIO.Weight = &w - } - - if s.Linux.Resources != nil && len(s.Linux.Resources.Devices) > 0 { - specResources.Devices = s.Linux.Resources.Devices + s.Linux.Resources.BlockIO.Weight = &w } + s.Linux.Resources.Pids = getPidsLimit(r) - s.Linux.Resources = specResources return nil } } @@ -984,6 +1022,15 @@ func WithResources(c *container.Container) coci.SpecOpts { // WithSysctls sets the container's sysctls func WithSysctls(c *container.Container) coci.SpecOpts { return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error { + if len(c.HostConfig.Sysctls) == 0 { + return nil + } + if s.Linux == nil { + s.Linux = &specs.Linux{} + } + if s.Linux.Sysctl == nil { + s.Linux.Sysctl = make(map[string]string) + } // We merge the sysctls injected above with the HostConfig (latter takes // precedence for backwards-compatibility reasons). for k, v := range c.HostConfig.Sysctls { @@ -996,6 +1043,9 @@ func WithSysctls(c *container.Container) coci.SpecOpts { // WithUser sets the container's user func WithUser(c *container.Container) coci.SpecOpts { return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error { + if s.Process == nil { + s.Process = &specs.Process{} + } var err error s.Process.User, err = getUser(c, c.Config.User) return err diff --git a/daemon/oci_opts.go b/daemon/oci_opts.go index c824999d50739..c8b1b633b6f08 100644 --- a/daemon/oci_opts.go +++ b/daemon/oci_opts.go @@ -13,6 +13,9 @@ import ( func WithConsoleSize(c *container.Container) coci.SpecOpts { return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) error { if c.HostConfig.ConsoleSize[0] > 0 || c.HostConfig.ConsoleSize[1] > 0 { + if s.Process == nil { + s.Process = &specs.Process{} + } s.Process.ConsoleSize = &specs.Box{ Height: c.HostConfig.ConsoleSize[0], Width: c.HostConfig.ConsoleSize[1], diff --git a/daemon/oci_utils.go b/daemon/oci_utils.go index 2d833502bd3a9..a47f7bab44bb2 100644 --- a/daemon/oci_utils.go +++ b/daemon/oci_utils.go @@ -9,7 +9,12 @@ func setLinuxDomainname(c *container.Container, s *specs.Spec) { // There isn't a field in the OCI for the NIS domainname, but luckily there // is a sysctl which has an identical effect to setdomainname(2) so there's // no explicit need for runtime support. - s.Linux.Sysctl = make(map[string]string) + if s.Linux == nil { + s.Linux = &specs.Linux{} + } + if s.Linux.Sysctl == nil { + s.Linux.Sysctl = make(map[string]string) + } if c.Config.Domainname != "" { s.Linux.Sysctl["kernel.domainname"] = c.Config.Domainname } diff --git a/daemon/seccomp_linux.go b/daemon/seccomp_linux.go index 8336b00392233..2e3c37818eb7b 100644 --- a/daemon/seccomp_linux.go +++ b/daemon/seccomp_linux.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/container" dconfig "github.com/docker/docker/daemon/config" "github.com/docker/docker/profiles/seccomp" + specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -31,6 +32,9 @@ func WithSeccomp(daemon *Daemon, c *container.Container) coci.SpecOpts { c.SeccompProfile = dconfig.SeccompProfileUnconfined return nil } + if s.Linux == nil { + s.Linux = &specs.Linux{} + } var err error switch { case c.SeccompProfile == dconfig.SeccompProfileDefault: diff --git a/oci/namespaces.go b/oci/namespaces.go index f32e489b4a27f..851edd61ef240 100644 --- a/oci/namespaces.go +++ b/oci/namespaces.go @@ -4,6 +4,9 @@ import specs "github.com/opencontainers/runtime-spec/specs-go" // RemoveNamespace removes the `nsType` namespace from OCI spec `s` func RemoveNamespace(s *specs.Spec, nsType specs.LinuxNamespaceType) { + if s.Linux == nil { + return + } for i, n := range s.Linux.Namespaces { if n.Type == nsType { s.Linux.Namespaces = append(s.Linux.Namespaces[:i], s.Linux.Namespaces[i+1:]...) diff --git a/oci/oci.go b/oci/oci.go index 2021ec3538fdd..864ccf5b60c41 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -20,6 +20,9 @@ var deviceCgroupRuleRegex = regexp.MustCompile("^([acb]) ([0-9]+|\\*):([0-9]+|\\ // SetCapabilities sets the provided capabilities on the spec // All capabilities are added if privileged is true. func SetCapabilities(s *specs.Spec, caplist []string) error { + if s.Process == nil { + s.Process = &specs.Process{} + } // setUser has already been executed here if s.Process.User.UID == 0 { s.Process.Capabilities = &specs.LinuxCapabilities{ diff --git a/pkg/rootless/specconv/specconv_linux.go b/pkg/rootless/specconv/specconv_linux.go index b706b5ca6a5f1..06f55ef13d789 100644 --- a/pkg/rootless/specconv/specconv_linux.go +++ b/pkg/rootless/specconv/specconv_linux.go @@ -40,11 +40,13 @@ func getCurrentOOMScoreAdj() int { func toRootless(spec *specs.Spec, v2Controllers []string, currentOOMScoreAdj int) error { if len(v2Controllers) == 0 { - // Remove cgroup settings. - spec.Linux.Resources = nil - spec.Linux.CgroupsPath = "" + if spec.Linux != nil { + // Remove cgroup settings. + spec.Linux.Resources = nil + spec.Linux.CgroupsPath = "" + } } else { - if spec.Linux.Resources != nil { + if spec.Linux != nil && spec.Linux.Resources != nil { m := make(map[string]struct{}) for _, s := range v2Controllers { m[s] = struct{}{} @@ -77,7 +79,7 @@ func toRootless(spec *specs.Spec, v2Controllers []string, currentOOMScoreAdj int } } - if spec.Process.OOMScoreAdj != nil && *spec.Process.OOMScoreAdj < currentOOMScoreAdj { + if spec.Process != nil && spec.Process.OOMScoreAdj != nil && *spec.Process.OOMScoreAdj < currentOOMScoreAdj { *spec.Process.OOMScoreAdj = currentOOMScoreAdj } @@ -110,6 +112,9 @@ func isHostNS(spec *specs.Spec, nsType specs.LinuxNamespaceType) (bool, error) { if strings.Contains(string(nsType), string(os.PathSeparator)) { return false, fmt.Errorf("unexpected namespace type %q", nsType) } + if spec.Linux == nil { + return false, nil + } for _, ns := range spec.Linux.Namespaces { if ns.Type == nsType { if ns.Path == "" { @@ -144,15 +149,17 @@ func bindMountHostProcfs(spec *specs.Spec) error { } } - // Remove ReadonlyPaths for /proc/* - newROP := spec.Linux.ReadonlyPaths[:0] - for _, s := range spec.Linux.ReadonlyPaths { - s = path.Clean(s) - if !strings.HasPrefix(s, "/proc/") { - newROP = append(newROP, s) + if spec.Linux != nil { + // Remove ReadonlyPaths for /proc/* + newROP := spec.Linux.ReadonlyPaths[:0] + for _, s := range spec.Linux.ReadonlyPaths { + s = path.Clean(s) + if !strings.HasPrefix(s, "/proc/") { + newROP = append(newROP, s) + } } + spec.Linux.ReadonlyPaths = newROP } - spec.Linux.ReadonlyPaths = newROP return nil } From 35a29c732890c2a1834cb7c0eac4f0979e571cd9 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Mon, 19 Jun 2023 05:47:09 +0200 Subject: [PATCH 089/293] builder: pass host-gateway IP as worker label We missed a case when parsing extra hosts from the dockerfile frontend so the build fails. To handle this case we need to set a dedicated worker label that contains the host gateway IP so clients like Buildx can just set the proper host:ip when parsing extra hosts that contain the special string "host-gateway". Signed-off-by: CrazyMax (cherry picked from commit 21e50b89c92666589780eba6f83ad0851d8e9235) --- builder/builder-next/controller.go | 11 +++++++++++ builder/builder-next/worker/label/label.go | 9 +++++++++ 2 files changed, 20 insertions(+) create mode 100644 builder/builder-next/worker/label/label.go diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index c84f788bc3ada..d21d88eeacbb0 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/builder/builder-next/exporter/mobyexporter" "github.com/docker/docker/builder/builder-next/imagerefchecker" mobyworker "github.com/docker/docker/builder/builder-next/worker" + wlabel "github.com/docker/docker/builder/builder-next/worker/label" "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/graphdriver" units "github.com/docker/go-units" @@ -95,6 +96,7 @@ func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt wo.GCPolicy = policy wo.RegistryHosts = opt.RegistryHosts + wo.Labels = getLabels(opt, wo.Labels) exec, err := newExecutor(opt.Root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile) if err != nil { @@ -325,6 +327,7 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt Layers: layers, Platforms: archutil.SupportedPlatforms(true), LeaseManager: lm, + Labels: getLabels(opt, nil), } wc := &worker.Controller{} @@ -411,3 +414,11 @@ func getEntitlements(conf config.BuilderConfig) []string { } return ents } + +func getLabels(opt Opt, labels map[string]string) map[string]string { + if labels == nil { + labels = make(map[string]string) + } + labels[wlabel.HostGatewayIP] = opt.DNSConfig.HostGatewayIP.String() + return labels +} diff --git a/builder/builder-next/worker/label/label.go b/builder/builder-next/worker/label/label.go new file mode 100644 index 0000000000000..f879720f8145e --- /dev/null +++ b/builder/builder-next/worker/label/label.go @@ -0,0 +1,9 @@ +package label + +// Pre-defined label keys similar to BuildKit ones +// https://github.com/moby/buildkit/blob/v0.11.6/worker/label/label.go#L3-L16 +const ( + prefix = "org.mobyproject.buildkit.worker.moby." + + HostGatewayIP = prefix + "host-gateway-ip" +) From 0556ba23a42205cdc6dbf56d0c861257977a1aed Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 5 May 2023 17:49:28 +0200 Subject: [PATCH 090/293] daemon: handleContainerExit(): use logrus.WithFields Use `WithFields()` instead of chaining multiple `WithField()` calls. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit de363f14049df35a9e9d357b1abf0544afa92b8f) Signed-off-by: Sebastiaan van Stijn --- daemon/monitor.go | 54 ++++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/daemon/monitor.go b/daemon/monitor.go index 90998d4643e3c..77a14f4aa14ae 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -39,7 +39,10 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine es, err := tsk.Delete(ctx) cancel() if err != nil { - logrus.WithError(err).WithField("container", c.ID).Warnf("failed to delete container from containerd") + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "container": c.ID, + }).Warn("failed to delete container from containerd") } else { exitStatus = container.ExitStatus{ ExitCode: int(es.ExitCode()), @@ -66,14 +69,15 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine execDuration := time.Since(c.StartedAt) restart, wait, err := c.RestartManager().ShouldRestart(uint32(exitStatus.ExitCode), daemonShutdown || c.HasBeenManuallyStopped, execDuration) if err != nil { - logrus.WithError(err). - WithField("container", c.ID). - WithField("restartCount", c.RestartCount). - WithField("exitStatus", exitStatus). - WithField("daemonShuttingDown", daemonShutdown). - WithField("hasBeenManuallyStopped", c.HasBeenManuallyStopped). - WithField("execDuration", execDuration). - Warn("ShouldRestart failed, container will not be restarted") + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "container": c.ID, + "restartCount": c.RestartCount, + "exitStatus": exitStatus, + "daemonShuttingDown": daemonShutdown, + "hasBeenManuallyStopped": c.HasBeenManuallyStopped, + "execDuration": execDuration, + }).Warn("ShouldRestart failed, container will not be restarted") restart = false } @@ -85,11 +89,12 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine if restart { c.RestartCount++ - logrus.WithField("container", c.ID). - WithField("restartCount", c.RestartCount). - WithField("exitStatus", exitStatus). - WithField("manualRestart", c.HasBeenManuallyRestarted). - Debug("Restarting container") + logrus.WithFields(logrus.Fields{ + "container": c.ID, + "restartCount": c.RestartCount, + "exitStatus": exitStatus, + "manualRestart": c.HasBeenManuallyRestarted, + }).Debug("Restarting container") c.SetRestarting(&exitStatus) } else { c.SetStopped(&exitStatus) @@ -188,9 +193,10 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei go func() { if _, err := execConfig.Process.Delete(context.Background()); err != nil { - logrus.WithError(err).WithFields(logrus.Fields{ - "container": ei.ContainerID, - "process": ei.ProcessID, + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "container": ei.ContainerID, + "process": ei.ProcessID, }).Warn("failed to delete process") } }() @@ -211,8 +217,10 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei if errdefs.IsNotFound(err) { // The container was started by not-docker and so could have been deleted by // not-docker before we got around to loading it from containerd. - logrus.WithField("container", c.ID).WithError(err). - Debug("could not load containerd container for start event") + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "container": c.ID, + }).Debug("could not load containerd container for start event") return nil } return err @@ -220,8 +228,10 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei tsk, err := ctr.Task(context.Background()) if err != nil { if errdefs.IsNotFound(err) { - logrus.WithField("container", c.ID).WithError(err). - Debug("failed to load task for externally-started container") + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "container": c.ID, + }).Debug("failed to load task for externally-started container") return nil } return err @@ -286,5 +296,5 @@ func (daemon *Daemon) autoRemove(c *container.Container) { return } - logrus.WithError(err).WithField("container", c.ID).Error("error removing container") + logrus.WithFields(logrus.Fields{logrus.ErrorKey: err, "container": c.ID}).Error("error removing container") } From 290fc0440c7bfc090021c08341eb1aa6bec6847d Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Thu, 22 Jun 2023 16:45:32 -0400 Subject: [PATCH 091/293] daemon: fix panic on failed exec start If an exec fails to start in such a way that containerd publishes an exit event for it, daemon.ProcessEvent will race daemon.ContainerExecStart in handling the failure. This race has been a long-standing bug, which was mostly harmless until 4bafaa00aa810dd17fde13e563def08f96fffc31. After that change, the daemon would dereference a nil pointer and crash if ProcessEvent won the race. Restore the status quo buggy behaviour by adding a check to skip the dereference if execConfig.Process is nil. Signed-off-by: Cory Snider (cherry picked from commit 3b28a24e97a3b4153b5c55bb6e757a1b6b326df4) Signed-off-by: Sebastiaan van Stijn --- daemon/monitor.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/daemon/monitor.go b/daemon/monitor.go index 77a14f4aa14ae..285a4890f3cbe 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -175,7 +175,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei // Remove the exec command from the container's store only and not the // daemon's store so that the exec command can be inspected. Remove it // before mutating execConfig to maintain the invariant that - // c.ExecCommands only contain execs in the Running state. + // c.ExecCommands only contains execs that have not exited. c.ExecCommands.Delete(execConfig.ID) execConfig.ExitCode = &ec @@ -191,15 +191,25 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei exitCode = ec - go func() { - if _, err := execConfig.Process.Delete(context.Background()); err != nil { - logrus.WithFields(logrus.Fields{ - logrus.ErrorKey: err, - "container": ei.ContainerID, - "process": ei.ProcessID, - }).Warn("failed to delete process") - } - }() + // If the exec failed at start in such a way that containerd + // publishes an exit event for it, we will race processing the event + // with daemon.ContainerExecStart() removing the exec from + // c.ExecCommands. If we win the race, we will find that there is no + // process to clean up. (And ContainerExecStart will clobber the + // exit code we set.) Prevent a nil-dereferenc panic in that + // situation to restore the status quo where this is merely a + // logical race condition. + if execConfig.Process != nil { + go func() { + if _, err := execConfig.Process.Delete(context.Background()); err != nil { + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "container": ei.ContainerID, + "process": ei.ProcessID, + }).Warn("failed to delete process") + } + }() + } } attributes := map[string]string{ "execID": ei.ProcessID, From 136893e33bde91871e76f3491d45ad82f9a78296 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Tue, 20 Jun 2023 14:14:55 -0400 Subject: [PATCH 092/293] daemon: fix double-unlock in health check probe Signed-off-by: Cory Snider (cherry picked from commit 786c9adaa2af6fbd5a369643de897cbeca884f3f) Signed-off-by: Sebastiaan van Stijn --- daemon/health.go | 1 - 1 file changed, 1 deletion(-) diff --git a/daemon/health.go b/daemon/health.go index fedc7efca7340..e9efebed6395e 100644 --- a/daemon/health.go +++ b/daemon/health.go @@ -160,7 +160,6 @@ func (p *cmdProbe) run(ctx context.Context, d *Daemon, cntr *container.Container info.Lock() defer info.Unlock() if info.ExitCode == nil { - info.Unlock() return 0, fmt.Errorf("healthcheck for container %s has no exit code", cntr.ID) } return *info.ExitCode, nil From 2f379ecfd6718155f476b044b7689464f6c6806b Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Fri, 23 Jun 2023 10:55:09 -0400 Subject: [PATCH 093/293] daemon: fix restoring container with missing task Before 4bafaa00aa810dd17fde13e563def08f96fffc31, if the daemon was killed while a container was running and the container shim is killed before the daemon is restarted, such as if the host system is hard-rebooted, the daemon would restore the container to the stopped state and set the exit code to 255. The aforementioned commit introduced a regression where the container's exit code would instead be set to 0. Fix the regression so that the exit code is once against set to 255 on restore. Signed-off-by: Cory Snider (cherry picked from commit 165dfd6c3eada774276a4db0038cddd9a28bc9a8) Signed-off-by: Sebastiaan van Stijn --- daemon/daemon.go | 2 + integration/container/daemon_linux_test.go | 90 ++++++++++++++++++---- testutil/daemon/daemon.go | 18 +++++ 3 files changed, 93 insertions(+), 17 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index a00a405dc18bf..4d76c57988884 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -420,6 +420,8 @@ func (daemon *Daemon) restore() error { if es != nil { ces.ExitCode = int(es.ExitCode()) ces.ExitedAt = es.ExitTime() + } else { + ces.ExitCode = 255 } c.SetStopped(&ces) daemon.Cleanup(c) diff --git a/integration/container/daemon_linux_test.go b/integration/container/daemon_linux_test.go index 5189c6bf53793..d1d6c61a9df5d 100644 --- a/integration/container/daemon_linux_test.go +++ b/integration/container/daemon_linux_test.go @@ -2,10 +2,8 @@ package container // import "github.com/docker/docker/integration/container" import ( "context" - "encoding/json" "fmt" "os" - "path/filepath" "strconv" "strings" "testing" @@ -19,6 +17,7 @@ import ( "golang.org/x/sys/unix" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" + "gotest.tools/v3/assert/opt" "gotest.tools/v3/skip" ) @@ -204,21 +203,10 @@ func TestRestartDaemonWithRestartingContainer(t *testing.T) { d.Stop(t) - configPath := filepath.Join(d.Root, "containers", id, "config.v2.json") - configBytes, err := os.ReadFile(configPath) - assert.NilError(t, err) - - var c realcontainer.Container - - assert.NilError(t, json.Unmarshal(configBytes, &c)) - - c.State = realcontainer.NewState() - c.SetRestarting(&realcontainer.ExitStatus{ExitCode: 1}) - c.HasBeenStartedBefore = true - - configBytes, err = json.Marshal(&c) - assert.NilError(t, err) - assert.NilError(t, os.WriteFile(configPath, configBytes, 0600)) + d.TamperWithContainerConfig(t, id, func(c *realcontainer.Container) { + c.SetRestarting(&realcontainer.ExitStatus{ExitCode: 1}) + c.HasBeenStartedBefore = true + }) d.Start(t) @@ -231,3 +219,71 @@ func TestRestartDaemonWithRestartingContainer(t *testing.T) { assert.NilError(t, err) } } + +// TestHardRestartWhenContainerIsRunning simulates a case where dockerd is +// killed while a container is running, and the container's task no longer +// exists when dockerd starts back up. This can happen if the system is +// hard-rebooted, for example. +// +// Regression test for moby/moby#45788 +func TestHardRestartWhenContainerIsRunning(t *testing.T) { + skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run") + skip.If(t, testEnv.DaemonInfo.OSType == "windows") + + t.Parallel() + + d := daemon.New(t) + defer d.Cleanup(t) + + d.StartWithBusybox(t, "--iptables=false") + defer d.Stop(t) + + ctx := context.Background() + client := d.NewClientT(t) + + // Just create the containers, no need to start them. + // We really want to make sure there is no process running when docker starts back up. + // We will manipulate the on disk state later. + nopolicy := container.Create(ctx, t, client, container.WithCmd("/bin/sh", "-c", "exit 1")) + onfailure := container.Create(ctx, t, client, container.WithRestartPolicy("on-failure"), container.WithCmd("/bin/sh", "-c", "sleep 60")) + + d.Stop(t) + + for _, id := range []string{nopolicy, onfailure} { + d.TamperWithContainerConfig(t, id, func(c *realcontainer.Container) { + c.SetRunning(nil, nil, true) + c.HasBeenStartedBefore = true + }) + } + + d.Start(t) + + t.Run("RestartPolicy=none", func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + inspect, err := client.ContainerInspect(ctx, nopolicy) + assert.NilError(t, err) + assert.Check(t, is.Equal(inspect.State.Status, "exited")) + assert.Check(t, is.Equal(inspect.State.ExitCode, 255)) + finishedAt, err := time.Parse(time.RFC3339Nano, inspect.State.FinishedAt) + if assert.Check(t, err) { + assert.Check(t, is.DeepEqual(finishedAt, time.Now(), opt.TimeWithThreshold(time.Minute))) + } + }) + + t.Run("RestartPolicy=on-failure", func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + inspect, err := client.ContainerInspect(ctx, onfailure) + assert.NilError(t, err) + assert.Check(t, is.Equal(inspect.State.Status, "running")) + assert.Check(t, is.Equal(inspect.State.ExitCode, 0)) + finishedAt, err := time.Parse(time.RFC3339Nano, inspect.State.FinishedAt) + if assert.Check(t, err) { + assert.Check(t, is.DeepEqual(finishedAt, time.Now(), opt.TimeWithThreshold(time.Minute))) + } + + stopTimeout := 0 + assert.Assert(t, client.ContainerStop(ctx, onfailure, containerapi.StopOptions{Timeout: &stopTimeout})) + }) +} diff --git a/testutil/daemon/daemon.go b/testutil/daemon/daemon.go index 2a1f5b3dc8298..98230960c6da0 100644 --- a/testutil/daemon/daemon.go +++ b/testutil/daemon/daemon.go @@ -16,6 +16,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/events" "github.com/docker/docker/client" + "github.com/docker/docker/container" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/testutil/request" @@ -825,6 +826,23 @@ func (d *Daemon) Info(t testing.TB) types.Info { return info } +// TamperWithContainerConfig modifies the on-disk config of a container. +func (d *Daemon) TamperWithContainerConfig(t testing.TB, containerID string, tamper func(*container.Container)) { + t.Helper() + + configPath := filepath.Join(d.Root, "containers", containerID, "config.v2.json") + configBytes, err := os.ReadFile(configPath) + assert.NilError(t, err) + + var c container.Container + assert.NilError(t, json.Unmarshal(configBytes, &c)) + c.State = container.NewState() + tamper(&c) + configBytes, err = json.Marshal(&c) + assert.NilError(t, err) + assert.NilError(t, os.WriteFile(configPath, configBytes, 0600)) +} + // cleanupRaftDir removes swarmkit wal files if present func cleanupRaftDir(t testing.TB, d *Daemon) { t.Helper() From 6055b07292c2b99d950f4a1dace3820c8eca0c24 Mon Sep 17 00:00:00 2001 From: Drew Erny Date: Fri, 23 Jun 2023 11:44:54 -0500 Subject: [PATCH 094/293] Fix missing Topology in NodeCSIInfo Added code to correctly retrieve and convert the Topology from the gRPC Swarm Node. Signed-off-by: Drew Erny (cherry picked from commit cdb1293eeab249e27ad55b00249a2424b77016e0) Signed-off-by: Sebastiaan van Stijn --- daemon/cluster/convert/node.go | 19 ++++++--- daemon/cluster/convert/node_test.go | 60 +++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 daemon/cluster/convert/node_test.go diff --git a/daemon/cluster/convert/node.go b/daemon/cluster/convert/node.go index 4ba9c626092e9..2019b8cfaded1 100644 --- a/daemon/cluster/convert/node.go +++ b/daemon/cluster/convert/node.go @@ -58,13 +58,20 @@ func NodeFromGRPC(n swarmapi.Node) types.Node { } for _, csi := range n.Description.CSIInfo { if csi != nil { + convertedInfo := types.NodeCSIInfo{ + PluginName: csi.PluginName, + NodeID: csi.NodeID, + MaxVolumesPerNode: csi.MaxVolumesPerNode, + } + + if csi.AccessibleTopology != nil { + convertedInfo.AccessibleTopology = &types.Topology{ + Segments: csi.AccessibleTopology.Segments, + } + } + node.Description.CSIInfo = append( - node.Description.CSIInfo, - types.NodeCSIInfo{ - PluginName: csi.PluginName, - NodeID: csi.NodeID, - MaxVolumesPerNode: csi.MaxVolumesPerNode, - }, + node.Description.CSIInfo, convertedInfo, ) } } diff --git a/daemon/cluster/convert/node_test.go b/daemon/cluster/convert/node_test.go new file mode 100644 index 0000000000000..2c003851af14f --- /dev/null +++ b/daemon/cluster/convert/node_test.go @@ -0,0 +1,60 @@ +package convert + +import ( + "testing" + + types "github.com/docker/docker/api/types/swarm" + swarmapi "github.com/moby/swarmkit/v2/api" + "gotest.tools/v3/assert" +) + +// TestNodeCSIInfoFromGRPC tests that conversion of the NodeCSIInfo from the +// gRPC to the Docker types is correct. +func TestNodeCSIInfoFromGRPC(t *testing.T) { + node := &swarmapi.Node{ + ID: "someID", + Description: &swarmapi.NodeDescription{ + CSIInfo: []*swarmapi.NodeCSIInfo{ + &swarmapi.NodeCSIInfo{ + PluginName: "plugin1", + NodeID: "p1n1", + MaxVolumesPerNode: 1, + }, + &swarmapi.NodeCSIInfo{ + PluginName: "plugin2", + NodeID: "p2n1", + MaxVolumesPerNode: 2, + AccessibleTopology: &swarmapi.Topology{ + Segments: map[string]string{ + "a": "1", + "b": "2", + }, + }, + }, + }, + }, + } + + expected := []types.NodeCSIInfo{ + { + PluginName: "plugin1", + NodeID: "p1n1", + MaxVolumesPerNode: 1, + }, + { + PluginName: "plugin2", + NodeID: "p2n1", + MaxVolumesPerNode: 2, + AccessibleTopology: &types.Topology{ + Segments: map[string]string{ + "a": "1", + "b": "2", + }, + }, + }, + } + + actual := NodeFromGRPC(*node) + + assert.DeepEqual(t, actual.Description.CSIInfo, expected) +} From 6424ae830b41cfcc0ed3b03688b03afd55d9d76b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 20 Jun 2023 12:33:24 +0200 Subject: [PATCH 095/293] Dockerfile: update buildx to v0.11.0 Update the version of buildx we use in the dev-container to v0.11.0; https://github.com/docker/buildx/releases/tag/v0.11.0 Full diff: https://github.com/docker/buildx/compare/v0.10.5..v0.11.0 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 4d831949a7a39e49f6d8f911d0b37826063a31fd) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1d183ddc5e2e6..9df32d1aecbbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG DOCKERCLI_VERSION=v24.0.2 # cli version used for integration-cli tests ARG DOCKERCLI_INTEGRATION_REPOSITORY="https://github.com/docker/cli.git" ARG DOCKERCLI_INTEGRATION_VERSION=v17.06.2-ce -ARG BUILDX_VERSION=0.10.5 +ARG BUILDX_VERSION=0.11.0 ARG SYSTEMD="false" ARG DEBIAN_FRONTEND=noninteractive From 96534f015dae5881f3f9f5096408acf7b20a9d6b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 26 Jun 2023 13:39:43 +0200 Subject: [PATCH 096/293] integration-cli: don't use pkg/homedir in test I'm considering deprecating the "Key()" utility, as it was only used in tests. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 0215a62d5b97d866e29812b3b1a000f23546d23b) Signed-off-by: Sebastiaan van Stijn --- integration-cli/docker_cli_run_unix_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/integration-cli/docker_cli_run_unix_test.go b/integration-cli/docker_cli_run_unix_test.go index 381776b2390fe..d1b5228c8cd0c 100644 --- a/integration-cli/docker_cli_run_unix_test.go +++ b/integration-cli/docker_cli_run_unix_test.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strconv" "strings" "syscall" @@ -22,7 +23,6 @@ import ( "github.com/docker/docker/client" "github.com/docker/docker/integration-cli/cli" "github.com/docker/docker/integration-cli/cli/build" - "github.com/docker/docker/pkg/homedir" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/sysinfo" "github.com/moby/sys/mount" @@ -251,7 +251,11 @@ func (s *DockerCLIRunSuite) TestRunAttachDetachFromConfig(c *testing.T) { os.Mkdir(dotDocker, 0600) tmpCfg := filepath.Join(dotDocker, "config.json") - c.Setenv(homedir.Key(), tmpDir) + if runtime.GOOS == "windows" { + c.Setenv("USERPROFILE", tmpDir) + } else { + c.Setenv("HOME", tmpDir) + } data := `{ "detachKeys": "ctrl-a,a" @@ -331,7 +335,11 @@ func (s *DockerCLIRunSuite) TestRunAttachDetachKeysOverrideConfig(c *testing.T) os.Mkdir(dotDocker, 0600) tmpCfg := filepath.Join(dotDocker, "config.json") - c.Setenv(homedir.Key(), tmpDir) + if runtime.GOOS == "windows" { + c.Setenv("USERPROFILE", tmpDir) + } else { + c.Setenv("HOME", tmpDir) + } data := `{ "detachKeys": "ctrl-e,e" From e84365f9675ddc375492b5fab090afff7b335bfe Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 26 Jun 2023 16:01:34 -0700 Subject: [PATCH 097/293] Skip cache lookup for "FROM scratch" in containerd Ideally, this should actually do a lookup across images that have no parent, but I wasn't 100% sure how to accomplish that so I opted for the smaller change of having `FROM scratch` builds not be cached for now. Signed-off-by: Tianon Gravi (cherry picked from commit 1741771b672b2418b4df2b2c6c0d4410ba12e65f) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/cache.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/daemon/containerd/cache.go b/daemon/containerd/cache.go index 8bd01768f8f9a..5e696c5967b9e 100644 --- a/daemon/containerd/cache.go +++ b/daemon/containerd/cache.go @@ -31,6 +31,12 @@ type imageCache struct { func (ic *imageCache) GetCache(parentID string, cfg *container.Config) (imageID string, err error) { ctx := context.TODO() + + if parentID == "" { + // TODO handle "parentless" image cache lookups ("FROM scratch") + return "", nil + } + parent, err := ic.c.GetImage(ctx, parentID, imagetype.GetImageOpts{}) if err != nil { return "", err From 85ad29966803dd17c4312d47b2dda7c042a95b55 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 16 Jun 2023 14:10:24 +0200 Subject: [PATCH 098/293] Dockerfile: make cli stages more resilient against unclean termination The Dockerfile in this repository performs many stages in parallel. If any of those stages fails to build (which could be due to networking congestion), other stages are also (forcibly?) terminated, which can cause an unclean shutdown. In some case, this can cause `git` to be terminated, leaving a `.lock` file behind in the cache mount. Retrying the build now will fail, and the only workaround is to clean the build-cache (which causes many stages to be built again, potentially triggering the problem again). > [dockercli-integration 3/3] RUN --mount=type=cache,id=dockercli-integration-git-linux/arm64/v8,target=./.git --mount=type=cache,target=/root/.cache/go-build,id=dockercli-integration-build-linux/arm64/v8 /download-or-build-cli.sh v17.06.2-ce https://github.com/docker/cli.git /build: #0 1.575 fatal: Unable to create '/go/src/github.com/docker/cli/.git/shallow.lock': File exists. #0 1.575 #0 1.575 Another git process seems to be running in this repository, e.g. #0 1.575 an editor opened by 'git commit'. Please make sure all processes #0 1.575 are terminated then try again. If it still fails, a git process #0 1.575 may have crashed in this repository earlier: #0 1.575 remove the file manually to continue. This patch: - Updates the Dockerfile to remove `.lock` files (`shallow.lock`, `index.lock`) that may have been left behind from previous builds. I put this code in the Dockerfile itself (not the script), as the script may be used in other situations outside of the Dockerfile (for which we cannot guarantee no other git session is active). - Adds a `docker --version` step to the stage; this is mostly to verify the build was successful (and to be consistent with other stages). Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 9f6dbbc7ea3e211e9a5d79120a8cb3c04bdf9767) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9df32d1aecbbe..3a80a248fa1ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -255,9 +255,11 @@ COPY hack/dockerfile/cli.sh /download-or-build-cli.sh ARG DOCKERCLI_REPOSITORY ARG DOCKERCLI_VERSION ARG TARGETPLATFORM -RUN --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,target=./.git \ +RUN --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,sharing=locked,target=./.git \ --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM \ - /download-or-build-cli.sh ${DOCKERCLI_VERSION} ${DOCKERCLI_REPOSITORY} /build + rm -f ./.git/*.lock \ + && /download-or-build-cli.sh ${DOCKERCLI_VERSION} ${DOCKERCLI_REPOSITORY} /build \ + && /build/docker --version FROM base AS dockercli-integration WORKDIR /go/src/github.com/docker/cli @@ -265,9 +267,11 @@ COPY hack/dockerfile/cli.sh /download-or-build-cli.sh ARG DOCKERCLI_INTEGRATION_REPOSITORY ARG DOCKERCLI_INTEGRATION_VERSION ARG TARGETPLATFORM -RUN --mount=type=cache,id=dockercli-integration-git-$TARGETPLATFORM,target=./.git \ +RUN --mount=type=cache,id=dockercli-integration-git-$TARGETPLATFORM,sharing=locked,target=./.git \ --mount=type=cache,target=/root/.cache/go-build,id=dockercli-integration-build-$TARGETPLATFORM \ - /download-or-build-cli.sh ${DOCKERCLI_INTEGRATION_VERSION} ${DOCKERCLI_INTEGRATION_REPOSITORY} /build + rm -f ./.git/*.lock \ + && /download-or-build-cli.sh ${DOCKERCLI_INTEGRATION_VERSION} ${DOCKERCLI_INTEGRATION_REPOSITORY} /build \ + && /build/docker --version # runc FROM base AS runc-src From d5e31e03b6e5f753d1ccf9bf525ac2c021623c9d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 27 Jun 2023 14:53:54 +0200 Subject: [PATCH 099/293] gha: Setup Runner: add missing sudo I think this may be missing a sudo (as all other operations do use sudo to access daemon.json); Run if [ ! -e /etc/docker/daemon.json ]; then if [ ! -e /etc/docker/daemon.json ]; then echo '{}' | tee /etc/docker/daemon.json >/dev/null fi DOCKERD_CONFIG=$(jq '.+{"experimental":true,"live-restore":true,"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' /etc/docker/daemon.json) sudo tee /etc/docker/daemon.json <<<"$DOCKERD_CONFIG" >/dev/null sudo service docker restart shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0} env: GO_VERSION: 1.20.5 GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 ITG_CLI_MATRIX_SIZE: 6 DOCKER_EXPERIMENTAL: 1 DOCKER_GRAPHDRIVER: overlay2 tee: /etc/docker/daemon.json: Permission denied Error: Process completed with exit code 1. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit d8bc5828cde9dbbbd7cea64ef1417589339597f2) Signed-off-by: Sebastiaan van Stijn --- .github/actions/setup-runner/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-runner/action.yml b/.github/actions/setup-runner/action.yml index d9e5211c23aa3..0c730ca8133c2 100644 --- a/.github/actions/setup-runner/action.yml +++ b/.github/actions/setup-runner/action.yml @@ -13,7 +13,7 @@ runs: shell: bash - run: | if [ ! -e /etc/docker/daemon.json ]; then - echo '{}' | tee /etc/docker/daemon.json >/dev/null + echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null fi DOCKERD_CONFIG=$(jq '.+{"experimental":true,"live-restore":true,"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' /etc/docker/daemon.json) sudo tee /etc/docker/daemon.json <<<"$DOCKERD_CONFIG" >/dev/null From c306276ab18ba62ea7b6d46961d3659e27df31a4 Mon Sep 17 00:00:00 2001 From: Vitor Anjos Date: Sun, 18 Jun 2023 14:21:12 -0300 Subject: [PATCH 100/293] remove name_to_handle_at(2) from filtered syscalls Signed-off-by: Vitor Anjos (cherry picked from commit fdc9b7ccebfde394a2c18d25a3d15d27ae72ed53) Signed-off-by: Bjorn Neergaard --- profiles/seccomp/default.json | 1 - profiles/seccomp/default_linux.go | 1 - 2 files changed, 2 deletions(-) diff --git a/profiles/seccomp/default.json b/profiles/seccomp/default.json index f361066a2f7ae..d20e2db5cabfd 100644 --- a/profiles/seccomp/default.json +++ b/profiles/seccomp/default.json @@ -601,7 +601,6 @@ "mount", "mount_setattr", "move_mount", - "name_to_handle_at", "open_tree", "perf_event_open", "quotactl", diff --git a/profiles/seccomp/default_linux.go b/profiles/seccomp/default_linux.go index 1ee7d7a808b08..561b2fc831c63 100644 --- a/profiles/seccomp/default_linux.go +++ b/profiles/seccomp/default_linux.go @@ -592,7 +592,6 @@ func DefaultProfile() *Seccomp { "mount", "mount_setattr", "move_mount", - "name_to_handle_at", "open_tree", "perf_event_open", "quotactl", From c24c37bd8a7de2ed96ac8f6aa2331324832a9f6a Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 14 Jun 2023 19:20:23 +0000 Subject: [PATCH 101/293] Restore active mount counts on live-restore When live-restoring a container the volume driver needs be notified that there is an active mount for the volume. Before this change the count is zero until the container stops and the uint64 overflows pretty much making it so the volume can never be removed until another daemon restart. Signed-off-by: Brian Goff (cherry picked from commit 647c2a6cdd86d79230df1bf690d0b6a2930d6db2) Signed-off-by: Bjorn Neergaard Signed-off-by: Sebastiaan van Stijn --- daemon/mounts.go | 12 +++++++++++ daemon/volumes.go | 7 ++++++ integration/daemon/daemon_test.go | 36 +++++++++++++++++++++++++++++++ volume/local/local.go | 28 ++++++++++++++++++++++++ volume/mounts/mounts.go | 29 +++++++++++++++++++++++++ volume/service/service.go | 16 ++++++++++++++ volume/service/store.go | 9 ++++++++ volume/volume.go | 10 +++++++++ 8 files changed, 147 insertions(+) diff --git a/daemon/mounts.go b/daemon/mounts.go index 383a38e7ebe2b..ad637df03d073 100644 --- a/daemon/mounts.go +++ b/daemon/mounts.go @@ -5,16 +5,28 @@ import ( "fmt" "strings" + "github.com/containerd/containerd/log" mounttypes "github.com/docker/docker/api/types/mount" "github.com/docker/docker/container" volumesservice "github.com/docker/docker/volume/service" + "github.com/sirupsen/logrus" ) func (daemon *Daemon) prepareMountPoints(container *container.Container) error { + alive := container.IsRunning() for _, config := range container.MountPoints { if err := daemon.lazyInitializeVolume(container.ID, config); err != nil { return err } + if alive { + log.G(context.TODO()).WithFields(logrus.Fields{ + "container": container.ID, + "volume": config.Volume.Name(), + }).Debug("Live-restoring volume for alive container") + if err := config.LiveRestore(context.TODO()); err != nil { + return err + } + } } return nil } diff --git a/daemon/volumes.go b/daemon/volumes.go index 6e17e221c671f..09871e7ebe182 100644 --- a/daemon/volumes.go +++ b/daemon/volumes.go @@ -21,6 +21,8 @@ import ( "github.com/sirupsen/logrus" ) +var _ volume.LiveRestorer = (*volumeWrapper)(nil) + type mounts []container.Mount // Len returns the number of mounts. Used in sorting. @@ -257,6 +259,7 @@ func (daemon *Daemon) VolumesService() *service.VolumesService { type volumeMounter interface { Mount(ctx context.Context, v *volumetypes.Volume, ref string) (string, error) Unmount(ctx context.Context, v *volumetypes.Volume, ref string) error + LiveRestoreVolume(ctx context.Context, v *volumetypes.Volume, ref string) error } type volumeWrapper struct { @@ -291,3 +294,7 @@ func (v *volumeWrapper) CreatedAt() (time.Time, error) { func (v *volumeWrapper) Status() map[string]interface{} { return v.v.Status } + +func (v *volumeWrapper) LiveRestoreVolume(ctx context.Context, ref string) error { + return v.s.LiveRestoreVolume(ctx, v.v, ref) +} diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index 7ecb2c0aa4793..9dcaed374757b 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -400,6 +400,42 @@ func testLiveRestoreVolumeReferences(t *testing.T) { runTest(t, "on-failure") runTest(t, "no") }) + + // Make sure that the local volume driver's mount ref count is restored + // Addresses https://github.com/moby/moby/issues/44422 + t.Run("local volume with mount options", func(t *testing.T) { + v, err := c.VolumeCreate(ctx, volume.CreateOptions{ + Driver: "local", + Name: "test-live-restore-volume-references-local", + DriverOpts: map[string]string{ + "type": "tmpfs", + "device": "tmpfs", + }, + }) + assert.NilError(t, err) + m := mount.Mount{ + Type: mount.TypeVolume, + Source: v.Name, + Target: "/foo", + } + cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("top")) + defer c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) + + d.Restart(t, "--live-restore", "--iptables=false") + + // Try to remove the volume + // This should fail since its used by a container + err = c.VolumeRemove(ctx, v.Name, false) + assert.ErrorContains(t, err, "volume is in use") + + // Remove that container which should free the references in the volume + err = c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) + assert.NilError(t, err) + + // Now we should be able to remove the volume + err = c.VolumeRemove(ctx, v.Name, false) + assert.NilError(t, err) + }) } func TestDaemonDefaultBridgeWithFixedCidrButNoBip(t *testing.T) { diff --git a/volume/local/local.go b/volume/local/local.go index 512e666eb8cf5..f156ea2339bc3 100644 --- a/volume/local/local.go +++ b/volume/local/local.go @@ -4,6 +4,7 @@ package local // import "github.com/docker/docker/volume/local" import ( + "context" "encoding/json" "os" "path/filepath" @@ -11,6 +12,7 @@ import ( "strings" "sync" + "github.com/containerd/containerd/log" "github.com/docker/docker/daemon/names" "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/idtools" @@ -35,6 +37,8 @@ var ( // This name is used to create the bind directory, so we need to avoid characters that // would make the path to escape the root directory. volumeNameRegex = names.RestrictedNamePattern + + _ volume.LiveRestorer = (*localVolume)(nil) ) type activeMount struct { @@ -296,14 +300,17 @@ func (v *localVolume) CachedPath() string { func (v *localVolume) Mount(id string) (string, error) { v.m.Lock() defer v.m.Unlock() + logger := log.G(context.TODO()).WithField("volume", v.name) if v.needsMount() { if !v.active.mounted { + logger.Debug("Mounting volume") if err := v.mount(); err != nil { return "", errdefs.System(err) } v.active.mounted = true } v.active.count++ + logger.WithField("active mounts", v.active).Debug("Decremented active mount count") } if err := v.postMount(); err != nil { return "", err @@ -316,6 +323,7 @@ func (v *localVolume) Mount(id string) (string, error) { func (v *localVolume) Unmount(id string) error { v.m.Lock() defer v.m.Unlock() + logger := log.G(context.TODO()).WithField("volume", v.name) // Always decrement the count, even if the unmount fails // Essentially docker doesn't care if this fails, it will send an error, but @@ -323,12 +331,14 @@ func (v *localVolume) Unmount(id string) error { // this volume can never be removed until a daemon restart occurs. if v.needsMount() { v.active.count-- + logger.WithField("active mounts", v.active).Debug("Decremented active mount count") } if v.active.count > 0 { return nil } + logger.Debug("Unmounting volume") return v.unmount() } @@ -369,6 +379,24 @@ func (v *localVolume) saveOpts() error { return nil } +// LiveRestoreVolume restores reference counts for mounts +// It is assumed that the volume is already mounted since this is only called for active, live-restored containers. +func (v *localVolume) LiveRestoreVolume(ctx context.Context, _ string) error { + v.m.Lock() + defer v.m.Unlock() + + if !v.needsMount() { + return nil + } + v.active.count++ + v.active.mounted = true + log.G(ctx).WithFields(logrus.Fields{ + "volume": v.name, + "active mounts": v.active, + }).Debugf("Live restored volume") + return nil +} + // getAddress finds out address/hostname from options func getAddress(opts string) string { for _, opt := range strings.Split(opts, ",") { diff --git a/volume/mounts/mounts.go b/volume/mounts/mounts.go index c441e51ed9018..bc90bb9def9de 100644 --- a/volume/mounts/mounts.go +++ b/volume/mounts/mounts.go @@ -1,17 +1,20 @@ package mounts // import "github.com/docker/docker/volume/mounts" import ( + "context" "fmt" "os" "path/filepath" "syscall" + "github.com/containerd/containerd/log" mounttypes "github.com/docker/docker/api/types/mount" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/volume" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // MountPoint is the intersection point between a volume and a container. It @@ -164,6 +167,32 @@ func (m *MountPoint) Setup(mountLabel string, rootIDs idtools.Identity, checkFun return m.Source, nil } +func (m *MountPoint) LiveRestore(ctx context.Context) error { + if m.Volume == nil { + logrus.Debug("No volume to restore") + return nil + } + + lrv, ok := m.Volume.(volume.LiveRestorer) + if !ok { + log.G(ctx).WithField("volume", m.Volume.Name()).Debugf("Volume does not support live restore: %T", m.Volume) + return nil + } + + id := m.ID + if id == "" { + id = stringid.GenerateRandomID() + } + + if err := lrv.LiveRestoreVolume(ctx, id); err != nil { + return errors.Wrapf(err, "error while restoring volume '%s'", m.Source) + } + + m.ID = id + m.active++ + return nil +} + // Path returns the path of a volume in a mount point. func (m *MountPoint) Path() string { if m.Volume != nil { diff --git a/volume/service/service.go b/volume/service/service.go index 7030b2a32bed4..adc7fdb239c03 100644 --- a/volume/service/service.go +++ b/volume/service/service.go @@ -5,6 +5,7 @@ import ( "strconv" "sync/atomic" + "github.com/containerd/containerd/log" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" volumetypes "github.com/docker/docker/api/types/volume" @@ -274,3 +275,18 @@ func (s *VolumesService) List(ctx context.Context, filter filters.Args) (volumes func (s *VolumesService) Shutdown() error { return s.vs.Shutdown() } + +// LiveRestoreVolume passes through the LiveRestoreVolume call to the volume if it is implemented +// otherwise it is a no-op. +func (s *VolumesService) LiveRestoreVolume(ctx context.Context, vol *volumetypes.Volume, ref string) error { + v, err := s.vs.Get(ctx, vol.Name, opts.WithGetDriver(vol.Driver)) + if err != nil { + return err + } + rlv, ok := v.(volume.LiveRestorer) + if !ok { + log.G(ctx).WithField("volume", vol.Name).Debugf("volume does not implement LiveRestoreVolume: %T", v) + return nil + } + return rlv.LiveRestoreVolume(ctx, ref) +} diff --git a/volume/service/store.go b/volume/service/store.go index 8926866e1c4d7..9250c869674c9 100644 --- a/volume/service/store.go +++ b/volume/service/store.go @@ -24,6 +24,8 @@ const ( volumeDataDir = "volumes" ) +var _ volume.LiveRestorer = (*volumeWrapper)(nil) + type volumeWrapper struct { volume.Volume labels map[string]string @@ -67,6 +69,13 @@ func (v volumeWrapper) CachedPath() string { return v.Volume.Path() } +func (v volumeWrapper) LiveRestoreVolume(ctx context.Context, ref string) error { + if vv, ok := v.Volume.(volume.LiveRestorer); ok { + return vv.LiveRestoreVolume(ctx, ref) + } + return nil +} + // StoreOpt sets options for a VolumeStore type StoreOpt func(store *VolumeStore) error diff --git a/volume/volume.go b/volume/volume.go index 61c8243979f11..2dcbdebe16f78 100644 --- a/volume/volume.go +++ b/volume/volume.go @@ -1,6 +1,7 @@ package volume // import "github.com/docker/docker/volume" import ( + "context" "time" ) @@ -60,6 +61,15 @@ type Volume interface { Status() map[string]interface{} } +// LiveRestorer is an optional interface that can be implemented by a volume driver +// It is used to restore any resources that are necessary for a volume to be used by a live-restored container +type LiveRestorer interface { + // LiveRestoreVolume allows a volume driver which implements this interface to restore any necessary resources (such as reference counting) + // This is called only after the daemon is restarted with live-restored containers + // It is called once per live-restored container. + LiveRestoreVolume(_ context.Context, ref string) error +} + // DetailedVolume wraps a Volume with user-defined labels, options, and cluster scope (e.g., `local` or `global`) type DetailedVolume interface { Labels() map[string]string From 806849eb6213d7cfe56d1c59f962be5d0d9845cd Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Wed, 28 Jun 2023 05:43:22 -0600 Subject: [PATCH 102/293] seccomp: add name_to_handle_at to allowlist Based on the analysis on [the previous PR][1]. [1]: https://github.com/moby/moby/pull/45766#pullrequestreview-1493908145 Signed-off-by: Bjorn Neergaard (cherry picked from commit b335e3d305be86bd28089a057d8be6a346445549) Signed-off-by: Bjorn Neergaard --- profiles/seccomp/default.json | 1 + profiles/seccomp/default_linux.go | 1 + 2 files changed, 2 insertions(+) diff --git a/profiles/seccomp/default.json b/profiles/seccomp/default.json index d20e2db5cabfd..cf785ef2c0ddb 100644 --- a/profiles/seccomp/default.json +++ b/profiles/seccomp/default.json @@ -237,6 +237,7 @@ "munlock", "munlockall", "munmap", + "name_to_handle_at", "nanosleep", "newfstatat", "_newselect", diff --git a/profiles/seccomp/default_linux.go b/profiles/seccomp/default_linux.go index 561b2fc831c63..c9ee04167789d 100644 --- a/profiles/seccomp/default_linux.go +++ b/profiles/seccomp/default_linux.go @@ -229,6 +229,7 @@ func DefaultProfile() *Seccomp { "munlock", "munlockall", "munmap", + "name_to_handle_at", "nanosleep", "newfstatat", "_newselect", From d9e097e328470ec89638a30c7ac214984cdf625e Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 29 Apr 2023 00:51:30 +0200 Subject: [PATCH 103/293] vendor: github.com/opencontainers/image-spec v1.1.0-rc3 full diff: https://github.com/opencontainers/image-spec/compare/3a7f492d3f1b...v1.1.0-rc3 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit b42e367045af7ca3f1ca5c7c718c265da76a75cd) Signed-off-by: Bjorn Neergaard --- daemon/containerd/image_builder.go | 16 +++++---- daemon/containerd/image_commit.go | 12 ++++--- daemon/containerd/image_import.go | 9 +++-- integration/image/pull_test.go | 9 ++--- vendor.mod | 2 +- vendor.sum | 4 +-- .../image-spec/specs-go/v1/annotations.go | 3 -- .../image-spec/specs-go/v1/artifact.go | 34 ------------------- .../image-spec/specs-go/v1/config.go | 34 ++++++------------- .../image-spec/specs-go/v1/manifest.go | 11 ++++++ .../image-spec/specs-go/v1/mediatype.go | 19 +++++++++-- .../image-spec/specs-go/version.go | 2 +- vendor/modules.txt | 4 +-- 13 files changed, 68 insertions(+), 91 deletions(-) delete mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/artifact.go diff --git a/daemon/containerd/image_builder.go b/daemon/containerd/image_builder.go index c92bd8bf3e88f..05338da2dada5 100644 --- a/daemon/containerd/image_builder.go +++ b/daemon/containerd/image_builder.go @@ -416,13 +416,15 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st // make an ocispec.Image from the docker/image.Image ociImgToCreate := ocispec.Image{ - Created: &imgToCreate.Created, - Author: imgToCreate.Author, - Architecture: imgToCreate.Architecture, - Variant: imgToCreate.Variant, - OS: imgToCreate.OS, - OSVersion: imgToCreate.OSVersion, - OSFeatures: imgToCreate.OSFeatures, + Created: &imgToCreate.Created, + Author: imgToCreate.Author, + Platform: ocispec.Platform{ + Architecture: imgToCreate.Architecture, + Variant: imgToCreate.Variant, + OS: imgToCreate.OS, + OSVersion: imgToCreate.OSVersion, + OSFeatures: imgToCreate.OSFeatures, + }, Config: ocispec.ImageConfig{ User: imgToCreate.Config.User, ExposedPorts: exposedPorts, diff --git a/daemon/containerd/image_commit.go b/daemon/containerd/image_commit.go index 632f28bc6bf5b..56941ee58625c 100644 --- a/daemon/containerd/image_commit.go +++ b/daemon/containerd/image_commit.go @@ -131,11 +131,13 @@ func generateCommitImageConfig(baseConfig ocispec.Image, diffID digest.Digest, o } logrus.Debugf("generateCommitImageConfig(): arch=%q, os=%q", arch, os) return ocispec.Image{ - Architecture: arch, - OS: os, - Created: &createdTime, - Author: opts.Author, - Config: containerConfigToOciImageConfig(opts.Config), + Platform: ocispec.Platform{ + Architecture: arch, + OS: os, + }, + Created: &createdTime, + Author: opts.Author, + Config: containerConfigToOciImageConfig(opts.Config), RootFS: ocispec.RootFS{ Type: "layers", DiffIDs: append(baseConfig.RootFS.DiffIDs, diffID), diff --git a/daemon/containerd/image_import.go b/daemon/containerd/image_import.go index 27b8b9ebea231..e4716642df360 100644 --- a/daemon/containerd/image_import.go +++ b/daemon/containerd/image_import.go @@ -86,11 +86,10 @@ func (i *ImageService) ImportImage(ctx context.Context, ref reference.Named, pla ociCfg := containerConfigToOciImageConfig(imageConfig) createdAt := time.Now() config := ocispec.Image{ - Architecture: platform.Architecture, - OS: platform.OS, - Created: &createdAt, - Author: "", - Config: ociCfg, + Platform: *platform, + Created: &createdAt, + Author: "", + Config: ociCfg, RootFS: ocispec.RootFS{ Type: "layers", DiffIDs: []digest.Digest{uncompressedDigest}, diff --git a/integration/image/pull_test.go b/integration/image/pull_test.go index a0ecb78d63360..f5ae78ad930b4 100644 --- a/integration/image/pull_test.go +++ b/integration/image/pull_test.go @@ -53,13 +53,10 @@ func createTestImage(ctx context.Context, t testing.TB, store content.Store) oci layerDigest := w.Digest() w.Close() - platform := platforms.DefaultSpec() - img := ocispec.Image{ - Architecture: platform.Architecture, - OS: platform.OS, - RootFS: ocispec.RootFS{Type: "layers", DiffIDs: []digest.Digest{layerDigest}}, - Config: ocispec.ImageConfig{WorkingDir: "/"}, + Platform: platforms.DefaultSpec(), + RootFS: ocispec.RootFS{Type: "layers", DiffIDs: []digest.Digest{layerDigest}}, + Config: ocispec.ImageConfig{WorkingDir: "/"}, } imgJSON, err := json.Marshal(img) assert.NilError(t, err) diff --git a/vendor.mod b/vendor.mod index 83725cc3e1a19..1e31d364a7d84 100644 --- a/vendor.mod +++ b/vendor.mod @@ -70,7 +70,7 @@ require ( github.com/moby/term v0.5.0 github.com/morikuni/aec v1.0.0 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b + github.com/opencontainers/image-spec v1.1.0-rc3 github.com/opencontainers/runc v1.1.7 github.com/opencontainers/runtime-spec v1.1.0-rc.2 github.com/opencontainers/selinux v1.11.0 diff --git a/vendor.sum b/vendor.sum index 5b63e18d63ec3..544dabc553fcb 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1137,8 +1137,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc10/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go index 6f9e6fd3abffe..e628920460461 100644 --- a/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go @@ -65,7 +65,4 @@ const ( // AnnotationArtifactDescription is the annotation key for the human readable description for the artifact. AnnotationArtifactDescription = "org.opencontainers.artifact.description" - - // AnnotationReferrersFiltersApplied is the annotation key for the comma separated list of filters applied by the registry in the referrers listing. - AnnotationReferrersFiltersApplied = "org.opencontainers.referrers.filtersApplied" ) diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/artifact.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/artifact.go deleted file mode 100644 index 03d76ce437ae0..0000000000000 --- a/vendor/github.com/opencontainers/image-spec/specs-go/v1/artifact.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2022 The Linux Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package v1 - -// Artifact describes an artifact manifest. -// This structure provides `application/vnd.oci.artifact.manifest.v1+json` mediatype when marshalled to JSON. -type Artifact struct { - // MediaType is the media type of the object this schema refers to. - MediaType string `json:"mediaType"` - - // ArtifactType is the IANA media type of the artifact this schema refers to. - ArtifactType string `json:"artifactType"` - - // Blobs is a collection of blobs referenced by this manifest. - Blobs []Descriptor `json:"blobs,omitempty"` - - // Subject (reference) is an optional link from the artifact to another manifest forming an association between the artifact and the other manifest. - Subject *Descriptor `json:"subject,omitempty"` - - // Annotations contains arbitrary metadata for the artifact manifest. - Annotations map[string]string `json:"annotations,omitempty"` -} diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/config.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/config.go index e6aa113f074e2..36b0aeb8f1fcb 100644 --- a/vendor/github.com/opencontainers/image-spec/specs-go/v1/config.go +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/config.go @@ -49,13 +49,15 @@ type ImageConfig struct { // StopSignal contains the system call signal that will be sent to the container to exit. StopSignal string `json:"StopSignal,omitempty"` - // ArgsEscaped `[Deprecated]` - This field is present only for legacy - // compatibility with Docker and should not be used by new image builders. - // It is used by Docker for Windows images to indicate that the `Entrypoint` - // or `Cmd` or both, contains only a single element array, that is a - // pre-escaped, and combined into a single string `CommandLine`. If `true` - // the value in `Entrypoint` or `Cmd` should be used as-is to avoid double - // escaping. + // ArgsEscaped + // + // Deprecated: This field is present only for legacy compatibility with + // Docker and should not be used by new image builders. It is used by Docker + // for Windows images to indicate that the `Entrypoint` or `Cmd` or both, + // contains only a single element array, that is a pre-escaped, and combined + // into a single string `CommandLine`. If `true` the value in `Entrypoint` or + // `Cmd` should be used as-is to avoid double escaping. + // https://github.com/opencontainers/image-spec/pull/892 ArgsEscaped bool `json:"ArgsEscaped,omitempty"` } @@ -95,22 +97,8 @@ type Image struct { // Author defines the name and/or email address of the person or entity which created and is responsible for maintaining the image. Author string `json:"author,omitempty"` - // Architecture is the CPU architecture which the binaries in this image are built to run on. - Architecture string `json:"architecture"` - - // Variant is the variant of the specified CPU architecture which image binaries are intended to run on. - Variant string `json:"variant,omitempty"` - - // OS is the name of the operating system which the image is built to run on. - OS string `json:"os"` - - // OSVersion is an optional field specifying the operating system - // version, for example on Windows `10.0.14393.1066`. - OSVersion string `json:"os.version,omitempty"` - - // OSFeatures is an optional field specifying an array of strings, - // each listing a required OS feature (for example on Windows `win32k`). - OSFeatures []string `json:"os.features,omitempty"` + // Platform describes the platform which the image in the manifest runs on. + Platform // Config defines the execution parameters which should be used as a base when running a container using the image. Config ImageConfig `json:"config,omitempty"` diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/manifest.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/manifest.go index 730a09359b1cb..4ce7b54ccdee0 100644 --- a/vendor/github.com/opencontainers/image-spec/specs-go/v1/manifest.go +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/manifest.go @@ -23,6 +23,9 @@ type Manifest struct { // MediaType specifies the type of this document data structure e.g. `application/vnd.oci.image.manifest.v1+json` MediaType string `json:"mediaType,omitempty"` + // ArtifactType specifies the IANA media type of artifact when the manifest is used for an artifact. + ArtifactType string `json:"artifactType,omitempty"` + // Config references a configuration object for a container, by digest. // The referenced configuration object is a JSON blob that the runtime uses to set up the container. Config Descriptor `json:"config"` @@ -36,3 +39,11 @@ type Manifest struct { // Annotations contains arbitrary metadata for the image manifest. Annotations map[string]string `json:"annotations,omitempty"` } + +// ScratchDescriptor is the descriptor of a blob with content of `{}`. +var ScratchDescriptor = Descriptor{ + MediaType: MediaTypeScratch, + Digest: `sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a`, + Size: 2, + Data: []byte(`{}`), +} diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/mediatype.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/mediatype.go index 935b481e3ed56..5dd31255eb0fe 100644 --- a/vendor/github.com/opencontainers/image-spec/specs-go/v1/mediatype.go +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/mediatype.go @@ -40,21 +40,36 @@ const ( // MediaTypeImageLayerNonDistributable is the media type for layers referenced by // the manifest but with distribution restrictions. + // + // Deprecated: Non-distributable layers are deprecated, and not recommended + // for future use. Implementations SHOULD NOT produce new non-distributable + // layers. + // https://github.com/opencontainers/image-spec/pull/965 MediaTypeImageLayerNonDistributable = "application/vnd.oci.image.layer.nondistributable.v1.tar" // MediaTypeImageLayerNonDistributableGzip is the media type for // gzipped layers referenced by the manifest but with distribution // restrictions. + // + // Deprecated: Non-distributable layers are deprecated, and not recommended + // for future use. Implementations SHOULD NOT produce new non-distributable + // layers. + // https://github.com/opencontainers/image-spec/pull/965 MediaTypeImageLayerNonDistributableGzip = "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip" // MediaTypeImageLayerNonDistributableZstd is the media type for zstd // compressed layers referenced by the manifest but with distribution // restrictions. + // + // Deprecated: Non-distributable layers are deprecated, and not recommended + // for future use. Implementations SHOULD NOT produce new non-distributable + // layers. + // https://github.com/opencontainers/image-spec/pull/965 MediaTypeImageLayerNonDistributableZstd = "application/vnd.oci.image.layer.nondistributable.v1.tar+zstd" // MediaTypeImageConfig specifies the media type for the image configuration. MediaTypeImageConfig = "application/vnd.oci.image.config.v1+json" - // MediaTypeArtifactManifest specifies the media type for a content descriptor. - MediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json" + // MediaTypeScratch specifies the media type for an unused blob containing the value `{}` + MediaTypeScratch = "application/vnd.oci.scratch.v1+json" ) diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/version.go b/vendor/github.com/opencontainers/image-spec/specs-go/version.go index 1afd590fe0b55..3d4119b4416e8 100644 --- a/vendor/github.com/opencontainers/image-spec/specs-go/version.go +++ b/vendor/github.com/opencontainers/image-spec/specs-go/version.go @@ -25,7 +25,7 @@ const ( VersionPatch = 0 // VersionDev indicates development branch. Releases will be empty string. - VersionDev = "-dev" + VersionDev = "-rc.3" ) // Version is the specification version that the package types support. diff --git a/vendor/modules.txt b/vendor/modules.txt index 8f948cd5d9166..4ae0e00ece0cc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -830,8 +830,8 @@ github.com/morikuni/aec ## explicit; go 1.13 github.com/opencontainers/go-digest github.com/opencontainers/go-digest/digestset -# github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b -## explicit; go 1.17 +# github.com/opencontainers/image-spec v1.1.0-rc3 +## explicit; go 1.18 github.com/opencontainers/image-spec/identity github.com/opencontainers/image-spec/specs-go github.com/opencontainers/image-spec/specs-go/v1 From b8ee9a7829ff4155bad6342295a7a1917b673cb1 Mon Sep 17 00:00:00 2001 From: Laura Brehm Date: Mon, 26 Jun 2023 16:14:38 +0200 Subject: [PATCH 104/293] c8d/images: handle images without manifests for default platform Signed-off-by: Laura Brehm (cherry picked from commit 6d3bcd8017544eeb4fca69bab2b4ad847c551d24) Resolved conflicts: daemon/containerd/image.go Signed-off-by: Bjorn Neergaard --- daemon/containerd/image.go | 44 ++++++++++++++++----------- daemon/containerd/image_history.go | 49 +++++++++++++++++------------- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index ee242132e1721..26dd20de29c95 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -2,14 +2,13 @@ package containerd import ( "context" - "encoding/json" "fmt" "regexp" + "sort" "strconv" "sync/atomic" "time" - "github.com/containerd/containerd/content" cerrdefs "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" cplatforms "github.com/containerd/containerd/platforms" @@ -33,7 +32,7 @@ var truncatedID = regexp.MustCompile(`^([a-f0-9]{4,64})$`) // GetImage returns an image corresponding to the image referred to by refOrID. func (i *ImageService) GetImage(ctx context.Context, refOrID string, options imagetype.GetImageOpts) (*image.Image, error) { - desc, err := i.resolveDescriptor(ctx, refOrID) + desc, err := i.resolveImage(ctx, refOrID) if err != nil { return nil, err } @@ -44,21 +43,32 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima } cs := i.client.ContentStore() - conf, err := containerdimages.Config(ctx, cs, desc, platform) - if err != nil { - return nil, err - } - imageConfigBytes, err := content.ReadBlob(ctx, cs, conf) + var presentImages []ocispec.Image + err = i.walkImageManifests(ctx, desc, func(img *ImageManifest) error { + conf, err := img.Config(ctx) + if err != nil { + return err + } + var ociimage ocispec.Image + if err := readConfig(ctx, cs, conf, &ociimage); err != nil { + return err + } + presentImages = append(presentImages, ociimage) + return nil + }) if err != nil { return nil, err } - - var ociimage ocispec.Image - if err := json.Unmarshal(imageConfigBytes, &ociimage); err != nil { - return nil, err + if len(presentImages) == 0 { + return nil, errdefs.NotFound(errors.New("failed to find image manifest")) } + sort.SliceStable(presentImages, func(i, j int) bool { + return platform.Less(presentImages[i].Platform, presentImages[j].Platform) + }) + ociimage := presentImages[0] + rootfs := image.NewRootFS() for _, id := range ociimage.RootFS.DiffIDs { rootfs.Append(layer.DiffID(id)) @@ -86,9 +96,9 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima }) } - img := image.NewImage(image.ID(desc.Digest)) + img := image.NewImage(image.ID(desc.Target.Digest)) img.V1Image = image.V1Image{ - ID: string(desc.Digest), + ID: string(desc.Target.Digest), OS: ociimage.OS, Architecture: ociimage.Architecture, Created: derefTimeSafely(ociimage.Created), @@ -110,12 +120,12 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima if options.Details { lastUpdated := time.Unix(0, 0) - size, err := i.size(ctx, desc, platform) + size, err := i.size(ctx, desc.Target, platform) if err != nil { return nil, err } - tagged, err := i.client.ImageService().List(ctx, "target.digest=="+desc.Digest.String()) + tagged, err := i.client.ImageService().List(ctx, "target.digest=="+desc.Target.Digest.String()) if err != nil { return nil, err } @@ -145,7 +155,7 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima } refs = append(refs, name) - digested, err := reference.WithDigest(reference.TrimNamed(name), desc.Digest) + digested, err := reference.WithDigest(reference.TrimNamed(name), desc.Target.Digest) if err != nil { // This could only happen if digest is invalid, but considering that // we get it from the Descriptor it's highly unlikely. diff --git a/daemon/containerd/image_history.go b/daemon/containerd/image_history.go index a2d0c11425b4c..a1fcd630c2ff5 100644 --- a/daemon/containerd/image_history.go +++ b/daemon/containerd/image_history.go @@ -2,13 +2,12 @@ package containerd import ( "context" - "encoding/json" + "sort" - "github.com/containerd/containerd/content" - containerdimages "github.com/containerd/containerd/images" cplatforms "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" imagetype "github.com/docker/docker/api/types/image" + "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/platforms" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -18,32 +17,39 @@ import ( // ImageHistory returns a slice of HistoryResponseItem structures for the // specified image name by walking the image lineage. func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imagetype.HistoryResponseItem, error) { - desc, err := i.resolveDescriptor(ctx, name) + desc, err := i.resolveImage(ctx, name) if err != nil { return nil, err } cs := i.client.ContentStore() - // TODO: pass the platform from the cli - conf, err := containerdimages.Config(ctx, cs, desc, platforms.AllPlatformsWithPreference(cplatforms.Default())) - if err != nil { - return nil, err - } + // TODO: pass platform in from the CLI + platform := platforms.AllPlatformsWithPreference(cplatforms.Default()) - diffIDs, err := containerdimages.RootFS(ctx, cs, conf) + var presentImages []ocispec.Image + err = i.walkImageManifests(ctx, desc, func(img *ImageManifest) error { + conf, err := img.Config(ctx) + if err != nil { + return err + } + var ociimage ocispec.Image + if err := readConfig(ctx, cs, conf, &ociimage); err != nil { + return err + } + presentImages = append(presentImages, ociimage) + return nil + }) if err != nil { return nil, err } - - blob, err := content.ReadBlob(ctx, cs, conf) - if err != nil { - return nil, err + if len(presentImages) == 0 { + return nil, errdefs.NotFound(errors.New("failed to find image manifest")) } - var image ocispec.Image - if err := json.Unmarshal(blob, &image); err != nil { - return nil, err - } + sort.SliceStable(presentImages, func(i, j int) bool { + return platform.Less(presentImages[i].Platform, presentImages[j].Platform) + }) + ociimage := presentImages[0] var ( history []*imagetype.HistoryResponseItem @@ -51,6 +57,7 @@ func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imaget ) s := i.client.SnapshotService(i.snapshotter) + diffIDs := ociimage.RootFS.DiffIDs for i := range diffIDs { chainID := identity.ChainID(diffIDs[0 : i+1]).String() @@ -62,7 +69,7 @@ func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imaget sizes = append(sizes, use.Size) } - for _, h := range image.History { + for _, h := range ociimage.History { size := int64(0) if !h.EmptyLayer { if len(sizes) == 0 { @@ -83,9 +90,9 @@ func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imaget } if len(history) != 0 { - history[0].ID = desc.Digest.String() + history[0].ID = desc.Target.Digest.String() - tagged, err := i.client.ImageService().List(ctx, "target.digest=="+desc.Digest.String()) + tagged, err := i.client.ImageService().List(ctx, "target.digest=="+desc.Target.Digest.String()) if err != nil { return nil, err } From ee29fd944bdc01c3e96eda8f295e2f72cd1c638d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 30 Jun 2023 01:50:42 +0200 Subject: [PATCH 105/293] gha: don't fail if no daemon.json is present CI failed sometimes if no daemon.json was present: Run sudo rm /etc/docker/daemon.json sudo rm /etc/docker/daemon.json sudo service docker restart docker version docker info shell: /usr/bin/bash -e {0} env: DESTDIR: ./build BUILDKIT_REPO: moby/buildkit BUILDKIT_TEST_DISABLE_FEATURES: cache_backend_azblob,cache_backend_s3,merge_diff BUILDKIT_REF: 798ad6b0ce9f2fe86dfb2b0277e6770d0b545871 rm: cannot remove '/etc/docker/daemon.json': No such file or directory Error: Process completed with exit code 1. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 264dbad43a7a8e971a4bbddceed5abeec9494c06) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/buildkit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index af336338d4c74..8dc35575ca6d4 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -106,7 +106,7 @@ jobs: - name: Update daemon.json run: | - sudo rm /etc/docker/daemon.json + sudo rm -f /etc/docker/daemon.json sudo service docker restart docker version docker info From cd44aba8db581d033de47bb9292e588f0d5b819d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 30 Jun 2023 18:28:55 +0200 Subject: [PATCH 106/293] [24.0] pkg/fileutils: switch to use containerd log pkg (very) partial backport of 74da6a6363d9d2991ea51ba44061a31a17b855cf and ab35df454d0666ef1214168938f952cc74c1d659 Signed-off-by: Sebastiaan van Stijn --- pkg/fileutils/fileutils_unix.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/fileutils/fileutils_unix.go b/pkg/fileutils/fileutils_unix.go index f782b4266aada..ab8e03a9a0f40 100644 --- a/pkg/fileutils/fileutils_unix.go +++ b/pkg/fileutils/fileutils_unix.go @@ -1,20 +1,20 @@ //go:build linux || freebsd -// +build linux freebsd package fileutils // import "github.com/docker/docker/pkg/fileutils" import ( + "context" "fmt" "os" - "github.com/sirupsen/logrus" + "github.com/containerd/containerd/log" ) // GetTotalUsedFds Returns the number of used File Descriptors by // reading it via /proc filesystem. func GetTotalUsedFds() int { if fds, err := os.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil { - logrus.Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err) + log.G(context.TODO()).Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err) } else { return len(fds) } From 01eb4835c9af29c02c9d35fbc18c608819f7b86c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 30 Jun 2023 01:31:51 +0200 Subject: [PATCH 107/293] pkg/fileutils: GetTotalUsedFds(): don't pretend to support FreeBSD Commit 8d56108ffb4e334600377c4bb4471eecec7b825c moved this function from the generic (no build-tags) fileutils.go to a unix file, adding "freebsd" to the build-tags. This likely was a wrong assumption (as other files had freebsd build-tags). FreeBSD's procfs does not mention `/proc//fd` in the manpage, and we don't test FreeBSD in CI, so let's drop it, and make this a Linux-only file. While updating also dropping the import-tag, as we're planning to move this file internal to the daemon. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 252e94f49952318ef777aefff9a6a8e9b71029e5) Signed-off-by: Sebastiaan van Stijn --- pkg/fileutils/{fileutils_unix.go => fileutils_linux.go} | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) rename pkg/fileutils/{fileutils_unix.go => fileutils_linux.go} (80%) diff --git a/pkg/fileutils/fileutils_unix.go b/pkg/fileutils/fileutils_linux.go similarity index 80% rename from pkg/fileutils/fileutils_unix.go rename to pkg/fileutils/fileutils_linux.go index ab8e03a9a0f40..37611c975ce49 100644 --- a/pkg/fileutils/fileutils_unix.go +++ b/pkg/fileutils/fileutils_linux.go @@ -1,6 +1,4 @@ -//go:build linux || freebsd - -package fileutils // import "github.com/docker/docker/pkg/fileutils" +package fileutils import ( "context" From 5dcea89ce12c6a5529aafcf743bd96b1e6f4d554 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 29 Jun 2023 15:36:22 +0200 Subject: [PATCH 108/293] pkg/fileutils: add BenchmarkGetTotalUsedFds go test -bench ^BenchmarkGetTotalUsedFds$ -run ^$ ./pkg/fileutils/ goos: linux goarch: arm64 pkg: github.com/docker/docker/pkg/fileutils BenchmarkGetTotalUsedFds-5 149272 7896 ns/op 945 B/op 20 allocs/op Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 03390be5fae9309bbaa27cfd73b6ed61babe9e71) Signed-off-by: Sebastiaan van Stijn --- pkg/fileutils/fileutils_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/fileutils/fileutils_test.go b/pkg/fileutils/fileutils_test.go index dc50eac30de49..7876ddf4b217e 100644 --- a/pkg/fileutils/fileutils_test.go +++ b/pkg/fileutils/fileutils_test.go @@ -240,3 +240,10 @@ func TestCreateIfNotExistsFile(t *testing.T) { t.Errorf("Should have been a file, seems it's not") } } + +func BenchmarkGetTotalUsedFds(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = GetTotalUsedFds() + } +} From bb50485dfdf47057e5a0d056dcea115406893be8 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 29 Jun 2023 15:50:02 +0200 Subject: [PATCH 109/293] pkg/fileutils: GetTotalUsedFds: reduce allocations Use File.Readdirnames instead of os.ReadDir, as we're only interested in the number of files, and results don't have to be sorted. Before: BenchmarkGetTotalUsedFds-5 149272 7896 ns/op 945 B/op 20 allocs/op After: BenchmarkGetTotalUsedFds-5 153517 7644 ns/op 408 B/op 10 allocs/op Signed-off-by: Sebastiaan van Stijn (cherry picked from commit eaa9494b71c046227264359beeaf80ddd4296ecf) Signed-off-by: Sebastiaan van Stijn --- pkg/fileutils/fileutils_linux.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/fileutils/fileutils_linux.go b/pkg/fileutils/fileutils_linux.go index 37611c975ce49..ea8faecec0966 100644 --- a/pkg/fileutils/fileutils_linux.go +++ b/pkg/fileutils/fileutils_linux.go @@ -3,6 +3,7 @@ package fileutils import ( "context" "fmt" + "io" "os" "github.com/containerd/containerd/log" @@ -11,10 +12,24 @@ import ( // GetTotalUsedFds Returns the number of used File Descriptors by // reading it via /proc filesystem. func GetTotalUsedFds() int { - if fds, err := os.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil { - log.G(context.TODO()).Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err) - } else { - return len(fds) + name := fmt.Sprintf("/proc/%d/fd", os.Getpid()) + f, err := os.Open(name) + if err != nil { + log.G(context.TODO()).WithError(err).Error("Error listing file descriptors") + return -1 } - return -1 + defer f.Close() + + var fdCount int + for { + names, err := f.Readdirnames(100) + fdCount += len(names) + if err == io.EOF { + break + } else if err != nil { + log.G(context.TODO()).WithError(err).Error("Error listing file descriptors") + return -1 + } + } + return fdCount } From aace62f6d354106443fde36c0399e75b8b452ca3 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 30 Jun 2023 00:46:46 +0200 Subject: [PATCH 110/293] pkg/fileutils: GetTotalUsedFds(): use fast-path for Kernel 6.2 and up Linux 6.2 and up (commit [f1f1f2569901ec5b9d425f2e91c09a0e320768f3][1]) provides a fast path for the number of open files for the process. From the [Linux docs][2]: > The number of open files for the process is stored in 'size' member of > `stat()` output for /proc//fd for fast access. [1]: https://github.com/torvalds/linux/commit/f1f1f2569901ec5b9d425f2e91c09a0e320768f3 [2]: https://docs.kernel.org/filesystems/proc.html#proc-pid-fd-list-of-symlinks-to-open-files This patch adds a fast-path for Kernels that support this, and falls back to the slow path if the Size fields is zero. Comparing on a Fedora 38 (kernel 6.2.9-300.fc38.x86_64): Before/After: go test -bench ^BenchmarkGetTotalUsedFds$ -run ^$ ./pkg/fileutils/ BenchmarkGetTotalUsedFds 57264 18595 ns/op 408 B/op 10 allocs/op BenchmarkGetTotalUsedFds 370392 3271 ns/op 40 B/op 3 allocs/op Note that the slow path has 1 more file-descriptor, due to the open file-handle for /proc//fd during the calculation. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ec79d0fc054a97b1d18adec794c6c30bfb11a488) Signed-off-by: Sebastiaan van Stijn --- pkg/fileutils/fileutils_linux.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/fileutils/fileutils_linux.go b/pkg/fileutils/fileutils_linux.go index ea8faecec0966..d8cd495f3f6fe 100644 --- a/pkg/fileutils/fileutils_linux.go +++ b/pkg/fileutils/fileutils_linux.go @@ -7,12 +7,27 @@ import ( "os" "github.com/containerd/containerd/log" + "golang.org/x/sys/unix" ) // GetTotalUsedFds Returns the number of used File Descriptors by // reading it via /proc filesystem. func GetTotalUsedFds() int { name := fmt.Sprintf("/proc/%d/fd", os.Getpid()) + + // Fast-path for Linux 6.2 (since [f1f1f2569901ec5b9d425f2e91c09a0e320768f3]). + // From the [Linux docs]: + // + // "The number of open files for the process is stored in 'size' member of + // stat() output for /proc//fd for fast access." + // + // [Linux docs]: https://docs.kernel.org/filesystems/proc.html#proc-pid-fd-list-of-symlinks-to-open-files: + // [f1f1f2569901ec5b9d425f2e91c09a0e320768f3]: https://github.com/torvalds/linux/commit/f1f1f2569901ec5b9d425f2e91c09a0e320768f3 + var stat unix.Stat_t + if err := unix.Stat(name, &stat); err == nil && stat.Size > 0 { + return int(stat.Size) + } + f, err := os.Open(name) if err != nil { log.G(context.TODO()).WithError(err).Error("Error listing file descriptors") @@ -31,5 +46,7 @@ func GetTotalUsedFds() int { return -1 } } + // Note that the slow path has 1 more file-descriptor, due to the open + // file-handle for /proc//fd during the calculation. return fdCount } From 42f3f7ed867be39f5481d607ed722eb9c3bb4da5 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 25 May 2023 01:38:24 +0200 Subject: [PATCH 111/293] c8d: ImageService.softImageDelete: use OCI and containerd constants Signed-off-by: Sebastiaan van Stijn (cherry picked from commit df5deab20b9d7477635496edff92aa57473e4153) Signed-off-by: Bjorn Neergaard --- daemon/containerd/soft_delete.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daemon/containerd/soft_delete.go b/daemon/containerd/soft_delete.go index f7514822149e1..c38883b805348 100644 --- a/daemon/containerd/soft_delete.go +++ b/daemon/containerd/soft_delete.go @@ -7,6 +7,7 @@ import ( containerdimages "github.com/containerd/containerd/images" "github.com/docker/docker/errdefs" "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -32,8 +33,8 @@ func (i *ImageService) softImageDelete(ctx context.Context, img containerdimages danglingImage := img danglingImage.Name = danglingImageName(img.Target.Digest) - delete(danglingImage.Labels, "io.containerd.image.name") - delete(danglingImage.Labels, "org.opencontainers.image.ref.name") + delete(danglingImage.Labels, containerdimages.AnnotationImageName) + delete(danglingImage.Labels, ocispec.AnnotationRefName) _, err = is.Create(context.Background(), danglingImage) From e0091d6616b552730687c3f64068d1140c9bada0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 25 May 2023 01:39:18 +0200 Subject: [PATCH 112/293] c8d: ImageService.softImageDelete: rename var that collided with import Signed-off-by: Sebastiaan van Stijn (cherry picked from commit f17c9e4aeb479307e6ee03a63663004b2bafb28c) Signed-off-by: Bjorn Neergaard --- daemon/containerd/soft_delete.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/daemon/containerd/soft_delete.go b/daemon/containerd/soft_delete.go index c38883b805348..ecba46e62a597 100644 --- a/daemon/containerd/soft_delete.go +++ b/daemon/containerd/soft_delete.go @@ -19,10 +19,10 @@ func (i *ImageService) softImageDelete(ctx context.Context, img containerdimages // If the image already exists, persist it as dangling image // but only if no other image has the same target. - digest := img.Target.Digest.String() - imgs, err := is.List(ctx, "target.digest=="+digest) + dgst := img.Target.Digest.String() + imgs, err := is.List(ctx, "target.digest=="+dgst) if err != nil { - return errdefs.System(errors.Wrapf(err, "failed to check if there are images targeting digest %s", digest)) + return errdefs.System(errors.Wrapf(err, "failed to check if there are images targeting digest %s", dgst)) } // From this point explicitly ignore the passed context From e2bade43e70a9a256e856c353129a89b44a4021f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 28 Jun 2023 14:25:11 +0200 Subject: [PATCH 113/293] testutil/environment: Add GetTestDanglingImageId MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit a96e6044cc5d8fb2095cc6ab74095ad3d4aee991) Signed-off-by: Bjorn Neergaard --- integration/image/inspect_test.go | 7 ++----- testutil/environment/special_images.go | 7 +++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/integration/image/inspect_test.go b/integration/image/inspect_test.go index 519e824c47f04..016e5c9f7661e 100644 --- a/integration/image/inspect_test.go +++ b/integration/image/inspect_test.go @@ -19,12 +19,9 @@ func TestImageInspectEmptyTagsAndDigests(t *testing.T) { client := testEnv.APIClient() ctx := context.Background() - danglingId := environment.DanglingImageIdGraphDriver - if testEnv.UsingSnapshotter() { - danglingId = environment.DanglingImageIdSnapshotter - } + danglingID := environment.GetTestDanglingImageId(testEnv) - inspect, raw, err := client.ImageInspectWithRaw(ctx, danglingId) + inspect, raw, err := client.ImageInspectWithRaw(ctx, danglingID) assert.NilError(t, err) // Must be a zero length array, not null. diff --git a/testutil/environment/special_images.go b/testutil/environment/special_images.go index b486e0498c762..a832cd7c3c645 100644 --- a/testutil/environment/special_images.go +++ b/testutil/environment/special_images.go @@ -5,3 +5,10 @@ const DanglingImageIdGraphDriver = "sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70 // The containerd image store identifies images by the ID of their manifest/manifest list. const DanglingImageIdSnapshotter = "sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456" + +func GetTestDanglingImageId(testEnv *Execution) string { + if testEnv.UsingSnapshotter() { + return DanglingImageIdSnapshotter + } + return DanglingImageIdGraphDriver +} From 8afe75ffa90cbda2572a6780eb5b10dbbbfbd377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 29 Jun 2023 14:36:22 +0200 Subject: [PATCH 114/293] c8d/softDelete: Extract ensureDanglingImage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 2b0655a71a3d67b4e750c41c0242ab23e1027a2b) Signed-off-by: Bjorn Neergaard --- daemon/containerd/soft_delete.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/daemon/containerd/soft_delete.go b/daemon/containerd/soft_delete.go index ecba46e62a597..9d26f7ade6057 100644 --- a/daemon/containerd/soft_delete.go +++ b/daemon/containerd/soft_delete.go @@ -30,19 +30,12 @@ func (i *ImageService) softImageDelete(ctx context.Context, img containerdimages // Create dangling image if this is the last image pointing to this target. if len(imgs) == 1 { - danglingImage := img - - danglingImage.Name = danglingImageName(img.Target.Digest) - delete(danglingImage.Labels, containerdimages.AnnotationImageName) - delete(danglingImage.Labels, ocispec.AnnotationRefName) - - _, err = is.Create(context.Background(), danglingImage) + err = i.ensureDanglingImage(context.Background(), img) // Error out in case we couldn't persist the old image. - // If it already exists, then just continue. - if err != nil && !cerrdefs.IsAlreadyExists(err) { + if err != nil { return errdefs.System(errors.Wrapf(err, "failed to create a dangling image for the replaced image %s with digest %s", - danglingImage.Name, danglingImage.Target.Digest.String())) + img.Name, img.Target.Digest.String())) } } @@ -57,6 +50,22 @@ func (i *ImageService) softImageDelete(ctx context.Context, img containerdimages return nil } +func (i *ImageService) ensureDanglingImage(ctx context.Context, from containerdimages.Image) error { + danglingImage := from + + danglingImage.Name = danglingImageName(from.Target.Digest) + delete(danglingImage.Labels, containerdimages.AnnotationImageName) + delete(danglingImage.Labels, ocispec.AnnotationRefName) + + _, err := i.client.ImageService().Create(context.Background(), danglingImage) + // If it already exists, then just continue. + if cerrdefs.IsAlreadyExists(err) { + return nil + } + + return err +} + func danglingImageName(digest digest.Digest) string { return "moby-dangling@" + digest.String() } From 8bf037b24676c63dd70aff313d74c527b1f47dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 30 Jun 2023 09:37:45 +0200 Subject: [PATCH 115/293] c8d/softDelete: Deep copy Labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So we don't override the original Labels in the passed image object. Signed-off-by: Paweł Gronowski (cherry picked from commit a6d5db3f9b97b818bcfd5f7a58876512c5a20f35) Signed-off-by: Bjorn Neergaard --- daemon/containerd/soft_delete.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/daemon/containerd/soft_delete.go b/daemon/containerd/soft_delete.go index 9d26f7ade6057..7ff6a044603da 100644 --- a/daemon/containerd/soft_delete.go +++ b/daemon/containerd/soft_delete.go @@ -53,9 +53,16 @@ func (i *ImageService) softImageDelete(ctx context.Context, img containerdimages func (i *ImageService) ensureDanglingImage(ctx context.Context, from containerdimages.Image) error { danglingImage := from + danglingImage.Labels = make(map[string]string) + for k, v := range from.Labels { + switch k { + case containerdimages.AnnotationImageName, ocispec.AnnotationRefName: + // Don't copy name labels. + default: + danglingImage.Labels[k] = v + } + } danglingImage.Name = danglingImageName(from.Target.Digest) - delete(danglingImage.Labels, containerdimages.AnnotationImageName) - delete(danglingImage.Labels, ocispec.AnnotationRefName) _, err := i.client.ImageService().Create(context.Background(), danglingImage) // If it already exists, then just continue. From 87778af7114606e9a3637abda8e3cb5724974076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 29 Jun 2023 15:38:08 +0200 Subject: [PATCH 116/293] c8d/prune: Exclude dangling tag of the images used by containers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit a93298d4dbea0158a51ed5d1901a7e8be7d2e07b) Signed-off-by: Bjorn Neergaard --- daemon/containerd/image_prune.go | 3 +++ integration/image/prune_test.go | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 integration/image/prune_test.go diff --git a/daemon/containerd/image_prune.go b/daemon/containerd/image_prune.go index d4c2052a13c3f..015df5b90b465 100644 --- a/daemon/containerd/image_prune.go +++ b/daemon/containerd/image_prune.go @@ -94,6 +94,9 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu var errs error // Exclude images used by existing containers for _, ctr := range containers { + // If the original image was deleted, make sure we don't delete the dangling image + delete(imagesToPrune, danglingImageName(ctr.ImageID.Digest())) + // Config.Image is the image reference passed by user. // For example: container created by `docker run alpine` will have Image="alpine" // Warning: This doesn't handle truncated ids: diff --git a/integration/image/prune_test.go b/integration/image/prune_test.go new file mode 100644 index 0000000000000..48fb07ce6fa99 --- /dev/null +++ b/integration/image/prune_test.go @@ -0,0 +1,33 @@ +package image + +import ( + "context" + "testing" + + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/testutil/environment" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" + "gotest.tools/v3/skip" +) + +// Regression test for: https://github.com/moby/moby/issues/45732 +func TestPruneDontDeleteUsedDangling(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType == "windows", "FIXME: hack/make/.build-empty-images doesn't run on Windows") + + defer setupTest(t)() + client := testEnv.APIClient() + ctx := context.Background() + + danglingID := environment.GetTestDanglingImageId(testEnv) + + container.Create(ctx, t, client, + container.WithImage(danglingID), + container.WithCmd("sleep", "60")) + + pruned, err := client.ImagesPrune(ctx, filters.NewArgs(filters.Arg("dangling", "true"))) + + assert.NilError(t, err) + assert.Check(t, is.Len(pruned.ImagesDeleted, 0)) +} From 016ad9b3e8e9a67aa8575f70a9ecf863b3883719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 29 Jun 2023 15:43:50 +0200 Subject: [PATCH 117/293] c8d/prune: Handle containers started from image id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If an image is only by id instead of its name, don't prune it completely. but only untag it and create a dangling image for it. Signed-off-by: Paweł Gronowski (cherry picked from commit e638351ef99adf35c33ea924f79ddee9561e1b20) Resolved conflicts: daemon/containerd/image_prune.go Signed-off-by: Bjorn Neergaard --- daemon/containerd/image_prune.go | 67 +++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/daemon/containerd/image_prune.go b/daemon/containerd/image_prune.go index 015df5b90b465..c9e78c80c4f7c 100644 --- a/daemon/containerd/image_prune.go +++ b/daemon/containerd/image_prune.go @@ -2,6 +2,7 @@ package containerd import ( "context" + "strings" cerrdefs "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" @@ -69,38 +70,50 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu return nil, err } + // How many images make reference to a particular target digest. + digestRefCount := map[digest.Digest]int{} + // Images considered for pruning. imagesToPrune := map[string]containerdimages.Image{} for _, img := range allImages { - if !danglingOnly || isDanglingImage(img) { - imagesToPrune[img.Name] = img - } - } + digestRefCount[img.Target.Digest] += 1 - // Apply filters - for name, img := range imagesToPrune { - filteredOut := !filterFunc(img) - logrus.WithFields(logrus.Fields{ - "image": name, - "filteredOut": filteredOut, - }).Debug("filtering image") + if !danglingOnly || isDanglingImage(img) { + canBePruned := filterFunc(img) + logrus.WithFields(logrus.Fields{ + "image": img.Name, + "canBePruned": canBePruned, + }).Debug("considering image for pruning") + + if canBePruned { + imagesToPrune[img.Name] = img + } - if filteredOut { - delete(imagesToPrune, name) } } - containers := i.containers.List() + // Image specified by digests that are used by containers. + usedDigests := map[digest.Digest]struct{}{} - var errs error // Exclude images used by existing containers - for _, ctr := range containers { + for _, ctr := range i.containers.List() { // If the original image was deleted, make sure we don't delete the dangling image delete(imagesToPrune, danglingImageName(ctr.ImageID.Digest())) // Config.Image is the image reference passed by user. - // For example: container created by `docker run alpine` will have Image="alpine" - // Warning: This doesn't handle truncated ids: - // `docker run 124c7d2` will have Image="124c7d270790" + // Config.ImageID is the resolved content digest based on the user's Config.Image. + // For example: container created by: + // `docker run alpine` will have Config.Image="alpine" + // `docker run 82d1e9d` will have Config.Image="82d1e9d" + // but both will have ImageID="sha256:82d1e9d7ed48a7523bdebc18cf6290bdb97b82302a8a9c27d4fe885949ea94d1" + imageDgst := ctr.ImageID.Digest() + + // If user didn't specify an explicit image, mark the digest as used. + normalizedImageID := "sha256:" + strings.TrimPrefix(ctr.Config.Image, "sha256:") + if strings.HasPrefix(imageDgst.String(), normalizedImageID) { + usedDigests[imageDgst] = struct{}{} + continue + } + ref, err := reference.ParseNormalizedNamed(ctr.Config.Image) logrus.WithFields(logrus.Fields{ "ctr": ctr.ID, @@ -109,12 +122,28 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu }).Debug("filtering container's image") if err == nil { + // If user provided a specific image name, exclude that image. name := reference.TagNameOnly(ref) delete(imagesToPrune, name.String()) } } + // Create dangling images for images that will be deleted but are still in use. + for _, img := range imagesToPrune { + dgst := img.Target.Digest + + digestRefCount[dgst] -= 1 + if digestRefCount[dgst] == 0 { + if _, isUsed := usedDigests[dgst]; isUsed { + if err := i.ensureDanglingImage(ctx, img); err != nil { + return &report, errors.Wrapf(err, "failed to create ensure dangling image for %s", img.Name) + } + } + } + } + possiblyDeletedConfigs := map[digest.Digest]struct{}{} + var errs error // Workaround for https://github.com/moby/buildkit/issues/3797 defer func() { From 457399013bfa84f01e3092222041a098ef164764 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 30 Jun 2023 19:32:26 +0200 Subject: [PATCH 118/293] vendor: github.com/containerd/cgroups/v3 v3.0.2 full diff: https://github.com/containerd/cgroups/compare/v3.0.1...v3.0.2 relevant changes: - cgroup2: only enable the cpuset controller if cpus or mems is specified - cgroup1 delete: proceed to the next subsystem when a cgroup is not found - Cgroup2: Reduce allocations for manager.Stat - Improve performance by for pid stats (cgroups1) re-using readuint - Reduce allocs in ReadUint64 by pre-allocating byte buffer - cgroup2: rm/simplify some code Signed-off-by: Sebastiaan van Stijn (cherry picked from commit f379af6d17d3aa3a1fc8890b1897933eb4458462) Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 +- .../containerd/cgroups/v3/README.md | 21 ++ .../containerd/cgroups/v3/cgroup1/blkio.go | 1 - .../containerd/cgroups/v3/cgroup1/cgroup.go | 10 +- .../containerd/cgroups/v3/cgroup1/control.go | 2 +- .../containerd/cgroups/v3/cgroup1/memory.go | 2 +- .../containerd/cgroups/v3/cgroup1/opts.go | 6 +- .../containerd/cgroups/v3/cgroup1/pids.go | 9 +- .../containerd/cgroups/v3/cgroup1/rdma.go | 1 - .../containerd/cgroups/v3/cgroup1/systemd.go | 5 +- .../containerd/cgroups/v3/cgroup1/utils.go | 19 +- .../containerd/cgroups/v3/cgroup1/v1.go | 2 +- .../cgroups/v3/cgroup2/devicefilter.go | 4 +- .../containerd/cgroups/v3/cgroup2/manager.go | 277 +++++++----------- .../containerd/cgroups/v3/cgroup2/utils.go | 164 +++++++---- vendor/modules.txt | 4 +- 17 files changed, 268 insertions(+), 265 deletions(-) diff --git a/vendor.mod b/vendor.mod index 1e31d364a7d84..04886abac8287 100644 --- a/vendor.mod +++ b/vendor.mod @@ -24,7 +24,7 @@ require ( github.com/aws/smithy-go v1.13.1 github.com/bsphere/le_go v0.0.0-20200109081728-fc06dab2caa8 github.com/cloudflare/cfssl v0.0.0-20180323000720-5d63dbd981b5 - github.com/containerd/cgroups/v3 v3.0.1 + github.com/containerd/cgroups/v3 v3.0.2 github.com/containerd/containerd v1.6.21 github.com/containerd/continuity v0.3.0 github.com/containerd/fifo v1.1.0 diff --git a/vendor.sum b/vendor.sum index 544dabc553fcb..d03c012de7c22 100644 --- a/vendor.sum +++ b/vendor.sum @@ -344,8 +344,8 @@ github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTF github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE= -github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw= +github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= +github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= diff --git a/vendor/github.com/containerd/cgroups/v3/README.md b/vendor/github.com/containerd/cgroups/v3/README.md index a90b87266e5ae..c7f37c612f3be 100644 --- a/vendor/github.com/containerd/cgroups/v3/README.md +++ b/vendor/github.com/containerd/cgroups/v3/README.md @@ -201,6 +201,27 @@ if err != nil { } ``` + +### Get and set cgroup type +```go +m, err := cgroup2.LoadSystemd("/", "my-cgroup-abc.slice") +if err != nil { + return err +} + +// https://www.kernel.org/doc/html/v5.0/admin-guide/cgroup-v2.html#threads +cgType, err := m.GetType() +if err != nil { + return err +} +fmt.Println(cgType) + +err = m.SetType(cgroup2.Threaded) +if err != nil { + return err +} +``` + ### Attention All static path should not include `/sys/fs/cgroup/` prefix, it should start with your own cgroups name diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/blkio.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/blkio.go index 9ee96e139655a..3be884c7e6c1a 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/blkio.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/blkio.go @@ -331,7 +331,6 @@ type deviceKey struct { // keyed by major and minor number. Since devices may be mapped multiple times, // we err on taking the first occurrence. func getDevices(r io.Reader) (map[deviceKey]string, error) { - var ( s = bufio.NewScanner(r) devices = make(map[deviceKey]string) diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/cgroup.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/cgroup.go index 2a31b8041f866..eae04f05bcbca 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/cgroup.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/cgroup.go @@ -41,7 +41,7 @@ func New(path Path, resources *specs.LinuxResources, opts ...InitOpts) (Cgroup, return nil, err } } - subsystems, err := config.hiearchy() + subsystems, err := config.hierarchy() if err != nil { return nil, err } @@ -79,7 +79,7 @@ func Load(path Path, opts ...InitOpts) (Cgroup, error) { } } var activeSubsystems []Subsystem - subsystems, err := config.hiearchy() + subsystems, err := config.hierarchy() if err != nil { return nil, err } @@ -158,7 +158,7 @@ func (c *cgroup) subsystemsFilter(subsystems ...Name) []Subsystem { return c.subsystems } - var filteredSubsystems = []Subsystem{} + filteredSubsystems := []Subsystem{} for _, s := range c.subsystems { for _, f := range subsystems { if s.Name() == f { @@ -259,6 +259,10 @@ func (c *cgroup) Delete() error { // kernel prevents cgroups with running process from being removed, check the tree is empty procs, err := c.processes(s.Name(), true, cgroupProcs) if err != nil { + // if the control group does not exist within a subsystem, then proceed to the next subsystem + if errors.Is(err, os.ErrNotExist) { + continue + } return err } if len(procs) > 0 { diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/control.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/control.go index 6cbf5323eabf2..8fee13d037a60 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/control.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/control.go @@ -28,7 +28,7 @@ type procType = string const ( cgroupProcs procType = "cgroup.procs" cgroupTasks procType = "tasks" - defaultDirPerm = 0755 + defaultDirPerm = 0o755 ) // defaultFilePerm is a var so that the test framework can change the filemode diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go index e84ec2b3c1d5d..caf5e9a7ebb0d 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/memory.go @@ -472,7 +472,7 @@ func (m *memoryController) memoryEvent(path string, event MemoryEvent) (uintptr, defer evtFile.Close() data := fmt.Sprintf("%d %d %s", efd, evtFile.Fd(), event.Arg()) evctlPath := filepath.Join(root, "cgroup.event_control") - if err := os.WriteFile(evctlPath, []byte(data), 0700); err != nil { + if err := os.WriteFile(evctlPath, []byte(data), 0o700); err != nil { unix.Close(efd) return 0, err } diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/opts.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/opts.go index 187e0f5eab3b6..3aa7f4fbbbe89 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/opts.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/opts.go @@ -36,13 +36,13 @@ type InitOpts func(*InitConfig) error type InitConfig struct { // InitCheck can be used to check initialization errors from the subsystem InitCheck InitCheck - hiearchy Hierarchy + hierarchy Hierarchy } func newInitConfig() *InitConfig { return &InitConfig{ InitCheck: RequireDevices, - hiearchy: Default, + hierarchy: Default, } } @@ -66,7 +66,7 @@ func RequireDevices(s Subsystem, _ Path, _ error) error { // The default list is coming from /proc/self/mountinfo. func WithHiearchy(h Hierarchy) InitOpts { return func(c *InitConfig) error { - c.hiearchy = h + c.hierarchy = h return nil } } diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/pids.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/pids.go index 9b5b263af54ae..31e2dda164d31 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/pids.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/pids.go @@ -20,7 +20,6 @@ import ( "os" "path/filepath" "strconv" - "strings" v1 "github.com/containerd/cgroups/v3/cgroup1/stats" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -67,16 +66,10 @@ func (p *pidsController) Stat(path string, stats *v1.Metrics) error { if err != nil { return err } - var max uint64 - maxData, err := os.ReadFile(filepath.Join(p.Path(path), "pids.max")) + max, err := readUint(filepath.Join(p.Path(path), "pids.max")) if err != nil { return err } - if maxS := strings.TrimSpace(string(maxData)); maxS != "max" { - if max, err = parseUint(maxS, 10, 64); err != nil { - return err - } - } stats.Pids = &v1.PidsStat{ Current: current, Limit: max, diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/rdma.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/rdma.go index 99299070cfb66..0a45ae08fbe21 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/rdma.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/rdma.go @@ -124,7 +124,6 @@ func toRdmaEntry(strEntries []string) []*v1.RdmaEntry { } func (p *rdmaController) Stat(path string, stats *v1.Metrics) error { - currentData, err := os.ReadFile(filepath.Join(p.Path(path), "rdma.current")) if err != nil { return err diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/systemd.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/systemd.go index d327effc8b977..335a255b833d7 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/systemd.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/systemd.go @@ -29,7 +29,7 @@ import ( const ( SystemdDbus Name = "systemd" - defaultSlice = "system.slice" + defaultSlice Name = "system.slice" ) var ( @@ -56,7 +56,7 @@ func Systemd() ([]Subsystem, error) { func Slice(slice, name string) Path { if slice == "" { - slice = defaultSlice + slice = string(defaultSlice) } return func(subsystem Name) (string, error) { return filepath.Join(slice, name), nil @@ -70,7 +70,6 @@ func NewSystemd(root string) (*SystemdController, error) { } type SystemdController struct { - mu sync.Mutex root string } diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/utils.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/utils.go index 8ae005dad2e5f..2b7d552001657 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/utils.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/utils.go @@ -18,6 +18,7 @@ package cgroup1 import ( "bufio" + "bytes" "fmt" "os" "path/filepath" @@ -131,11 +132,25 @@ func hugePageSizes() ([]string, error) { } func readUint(path string) (uint64, error) { - v, err := os.ReadFile(path) + f, err := os.Open(path) if err != nil { return 0, err } - return parseUint(strings.TrimSpace(string(v)), 10, 64) + defer f.Close() + + // We should only need 20 bytes for the max uint64, but for a nice power of 2 + // lets use 32. + b := make([]byte, 32) + n, err := f.Read(b) + if err != nil { + return 0, err + } + s := string(bytes.TrimSpace(b[:n])) + if s == "max" { + // Return 0 for the max value to maintain backward compatibility. + return 0, nil + } + return parseUint(s, 10, 64) } func parseUint(s string, base, bitSize int) (uint64, error) { diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup1/v1.go b/vendor/github.com/containerd/cgroups/v3/cgroup1/v1.go index d4c7db6f20177..ce025bbd98bec 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup1/v1.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup1/v1.go @@ -45,7 +45,7 @@ func Default() ([]Subsystem, error) { } // v1MountPoint returns the mount point where the cgroup -// mountpoints are mounted in a single hiearchy +// mountpoints are mounted in a single hierarchy func v1MountPoint() (string, error) { f, err := os.Open("/proc/self/mountinfo") if err != nil { diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup2/devicefilter.go b/vendor/github.com/containerd/cgroups/v3/cgroup2/devicefilter.go index 3a73ab1050fc0..0cd5f7f3ddff1 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup2/devicefilter.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup2/devicefilter.go @@ -167,7 +167,7 @@ func (p *program) appendDevice(dev specs.LinuxDeviceCgroup) error { } p.insts = append(p.insts, acceptBlock(dev.Allow)...) // set blockSym to the first instruction we added in this iteration - p.insts[prevBlockLastIdx+1] = p.insts[prevBlockLastIdx+1].Sym(blockSym) + p.insts[prevBlockLastIdx+1] = p.insts[prevBlockLastIdx+1].WithSymbol(blockSym) p.blockID++ return nil } @@ -180,7 +180,7 @@ func (p *program) finalize() (asm.Instructions, error) { blockSym := fmt.Sprintf("block-%d", p.blockID) p.insts = append(p.insts, // R0 <- 0 - asm.Mov.Imm32(asm.R0, 0).Sym(blockSym), + asm.Mov.Imm32(asm.R0, 0).WithSymbol(blockSym), asm.Return(), ) p.blockID = -1 diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go b/vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go index fc9fcf4533ef1..4a4292d5fcbfc 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup2/manager.go @@ -21,13 +21,11 @@ import ( "context" "errors" "fmt" - "io" "math" "os" "path/filepath" "strconv" "strings" - "syscall" "time" "github.com/containerd/cgroups/v3/cgroup2/stats" @@ -43,13 +41,12 @@ const ( subtreeControl = "cgroup.subtree_control" controllersFile = "cgroup.controllers" killFile = "cgroup.kill" + typeFile = "cgroup.type" defaultCgroup2Path = "/sys/fs/cgroup" defaultSlice = "system.slice" ) -var ( - canDelegate bool -) +var canDelegate bool type Event struct { Low uint64 @@ -99,7 +96,9 @@ func (r *Resources) Values() (o []Value) { func (r *Resources) EnabledControllers() (c []string) { if r.CPU != nil { c = append(c, "cpu") - c = append(c, "cpuset") + if r.CPU.Cpus != "" || r.CPU.Mems != "" { + c = append(c, "cpuset") + } } if r.Memory != nil { c = append(c, "memory") @@ -238,6 +237,35 @@ func setResources(path string, resources *Resources) error { return nil } +// CgroupType represents the types a cgroup can be. +type CgroupType string + +const ( + Domain CgroupType = "domain" + Threaded CgroupType = "threaded" +) + +func (c *Manager) GetType() (CgroupType, error) { + val, err := os.ReadFile(filepath.Join(c.path, typeFile)) + if err != nil { + return "", err + } + trimmed := strings.TrimSpace(string(val)) + return CgroupType(trimmed), nil +} + +func (c *Manager) SetType(cgType CgroupType) error { + // NOTE: We could abort if cgType != Threaded here as currently + // it's not possible to revert back to domain, but not sure + // it's worth being that opinionated, especially if that may + // ever change. + v := Value{ + filename: typeFile, + value: string(cgType), + } + return writeValues(c.path, []Value{v}) +} + func (c *Manager) RootControllers() ([]string, error) { b, err := os.ReadFile(filepath.Join(c.unifiedMountpoint, controllersFile)) if err != nil { @@ -492,17 +520,15 @@ func (c *Manager) MoveTo(destination *Manager) error { return nil } -var singleValueFiles = []string{ - "pids.current", - "pids.max", -} - func (c *Manager) Stat() (*stats.Metrics, error) { controllers, err := c.Controllers() if err != nil { return nil, err } - out := make(map[string]interface{}) + // Sizing this avoids an allocation to increase the map at runtime; + // currently the default bucket size is 8 and we put 40+ elements + // in it so we'd always end up allocating. + out := make(map[string]uint64, 50) for _, controller := range controllers { switch controller { case "cpu", "memory": @@ -514,66 +540,58 @@ func (c *Manager) Stat() (*stats.Metrics, error) { } } } - for _, name := range singleValueFiles { - if err := readSingleFile(c.path, name, out); err != nil { - if os.IsNotExist(err) { - continue - } - return nil, err - } - } - memoryEvents := make(map[string]interface{}) + memoryEvents := make(map[string]uint64) if err := readKVStatsFile(c.path, "memory.events", memoryEvents); err != nil { if !os.IsNotExist(err) { return nil, err } } - var metrics stats.Metrics + var metrics stats.Metrics metrics.Pids = &stats.PidsStat{ - Current: getPidValue("pids.current", out), - Limit: getPidValue("pids.max", out), + Current: getStatFileContentUint64(filepath.Join(c.path, "pids.current")), + Limit: getStatFileContentUint64(filepath.Join(c.path, "pids.max")), } metrics.CPU = &stats.CPUStat{ - UsageUsec: getUint64Value("usage_usec", out), - UserUsec: getUint64Value("user_usec", out), - SystemUsec: getUint64Value("system_usec", out), - NrPeriods: getUint64Value("nr_periods", out), - NrThrottled: getUint64Value("nr_throttled", out), - ThrottledUsec: getUint64Value("throttled_usec", out), + UsageUsec: out["usage_usec"], + UserUsec: out["user_usec"], + SystemUsec: out["system_usec"], + NrPeriods: out["nr_periods"], + NrThrottled: out["nr_throttled"], + ThrottledUsec: out["throttled_usec"], } metrics.Memory = &stats.MemoryStat{ - Anon: getUint64Value("anon", out), - File: getUint64Value("file", out), - KernelStack: getUint64Value("kernel_stack", out), - Slab: getUint64Value("slab", out), - Sock: getUint64Value("sock", out), - Shmem: getUint64Value("shmem", out), - FileMapped: getUint64Value("file_mapped", out), - FileDirty: getUint64Value("file_dirty", out), - FileWriteback: getUint64Value("file_writeback", out), - AnonThp: getUint64Value("anon_thp", out), - InactiveAnon: getUint64Value("inactive_anon", out), - ActiveAnon: getUint64Value("active_anon", out), - InactiveFile: getUint64Value("inactive_file", out), - ActiveFile: getUint64Value("active_file", out), - Unevictable: getUint64Value("unevictable", out), - SlabReclaimable: getUint64Value("slab_reclaimable", out), - SlabUnreclaimable: getUint64Value("slab_unreclaimable", out), - Pgfault: getUint64Value("pgfault", out), - Pgmajfault: getUint64Value("pgmajfault", out), - WorkingsetRefault: getUint64Value("workingset_refault", out), - WorkingsetActivate: getUint64Value("workingset_activate", out), - WorkingsetNodereclaim: getUint64Value("workingset_nodereclaim", out), - Pgrefill: getUint64Value("pgrefill", out), - Pgscan: getUint64Value("pgscan", out), - Pgsteal: getUint64Value("pgsteal", out), - Pgactivate: getUint64Value("pgactivate", out), - Pgdeactivate: getUint64Value("pgdeactivate", out), - Pglazyfree: getUint64Value("pglazyfree", out), - Pglazyfreed: getUint64Value("pglazyfreed", out), - ThpFaultAlloc: getUint64Value("thp_fault_alloc", out), - ThpCollapseAlloc: getUint64Value("thp_collapse_alloc", out), + Anon: out["anon"], + File: out["file"], + KernelStack: out["kernel_stack"], + Slab: out["slab"], + Sock: out["sock"], + Shmem: out["shmem"], + FileMapped: out["file_mapped"], + FileDirty: out["file_dirty"], + FileWriteback: out["file_writeback"], + AnonThp: out["anon_thp"], + InactiveAnon: out["inactive_anon"], + ActiveAnon: out["active_anon"], + InactiveFile: out["inactive_file"], + ActiveFile: out["active_file"], + Unevictable: out["unevictable"], + SlabReclaimable: out["slab_reclaimable"], + SlabUnreclaimable: out["slab_unreclaimable"], + Pgfault: out["pgfault"], + Pgmajfault: out["pgmajfault"], + WorkingsetRefault: out["workingset_refault"], + WorkingsetActivate: out["workingset_activate"], + WorkingsetNodereclaim: out["workingset_nodereclaim"], + Pgrefill: out["pgrefill"], + Pgscan: out["pgscan"], + Pgsteal: out["pgsteal"], + Pgactivate: out["pgactivate"], + Pgdeactivate: out["pgdeactivate"], + Pglazyfree: out["pglazyfree"], + Pglazyfreed: out["pglazyfreed"], + ThpFaultAlloc: out["thp_fault_alloc"], + ThpCollapseAlloc: out["thp_collapse_alloc"], Usage: getStatFileContentUint64(filepath.Join(c.path, "memory.current")), UsageLimit: getStatFileContentUint64(filepath.Join(c.path, "memory.max")), SwapUsage: getStatFileContentUint64(filepath.Join(c.path, "memory.swap.current")), @@ -581,11 +599,11 @@ func (c *Manager) Stat() (*stats.Metrics, error) { } if len(memoryEvents) > 0 { metrics.MemoryEvents = &stats.MemoryEvents{ - Low: getUint64Value("low", memoryEvents), - High: getUint64Value("high", memoryEvents), - Max: getUint64Value("max", memoryEvents), - Oom: getUint64Value("oom", memoryEvents), - OomKill: getUint64Value("oom_kill", memoryEvents), + Low: memoryEvents["low"], + High: memoryEvents["high"], + Max: memoryEvents["max"], + Oom: memoryEvents["oom"], + OomKill: memoryEvents["oom_kill"], } } metrics.Io = &stats.IOStat{Usage: readIoStats(c.path)} @@ -598,56 +616,7 @@ func (c *Manager) Stat() (*stats.Metrics, error) { return &metrics, nil } -func getUint64Value(key string, out map[string]interface{}) uint64 { - v, ok := out[key] - if !ok { - return 0 - } - switch t := v.(type) { - case uint64: - return t - } - return 0 -} - -func getPidValue(key string, out map[string]interface{}) uint64 { - v, ok := out[key] - if !ok { - return 0 - } - switch t := v.(type) { - case uint64: - return t - case string: - if t == "max" { - return math.MaxUint64 - } - } - return 0 -} - -func readSingleFile(path string, file string, out map[string]interface{}) error { - f, err := os.Open(filepath.Join(path, file)) - if err != nil { - return err - } - defer f.Close() - data, err := io.ReadAll(f) - if err != nil { - return err - } - s := strings.TrimSpace(string(data)) - v, err := parseUint(s, 10, 64) - if err != nil { - // if we cannot parse as a uint, parse as a string - out[file] = s - return nil - } - out[file] = v - return nil -} - -func readKVStatsFile(path string, file string, out map[string]interface{}) error { +func readKVStatsFile(path string, file string, out map[string]uint64) error { f, err := os.Open(filepath.Join(path, file)) if err != nil { return err @@ -692,16 +661,12 @@ func (c *Manager) freeze(path string, state State) error { func (c *Manager) isCgroupEmpty() bool { // In case of any error we return true so that we exit and don't leak resources - out := make(map[string]interface{}) + out := make(map[string]uint64) if err := readKVStatsFile(c.path, "cgroup.events", out); err != nil { return true } if v, ok := out["populated"]; ok { - populated, ok := v.(uint64) - if !ok { - return true - } - return populated == 0 + return v == 0 } return true } @@ -709,19 +674,19 @@ func (c *Manager) isCgroupEmpty() bool { // MemoryEventFD returns inotify file descriptor and 'memory.events' inotify watch descriptor func (c *Manager) MemoryEventFD() (int, uint32, error) { fpath := filepath.Join(c.path, "memory.events") - fd, err := syscall.InotifyInit() + fd, err := unix.InotifyInit() if err != nil { return 0, 0, errors.New("failed to create inotify fd") } - wd, err := syscall.InotifyAddWatch(fd, fpath, unix.IN_MODIFY) + wd, err := unix.InotifyAddWatch(fd, fpath, unix.IN_MODIFY) if err != nil { - syscall.Close(fd) + unix.Close(fd) return 0, 0, fmt.Errorf("failed to add inotify watch for %q: %w", fpath, err) } // monitor to detect process exit/cgroup deletion evpath := filepath.Join(c.path, "cgroup.events") - if _, err = syscall.InotifyAddWatch(fd, evpath, unix.IN_MODIFY); err != nil { - syscall.Close(fd) + if _, err = unix.InotifyAddWatch(fd, evpath, unix.IN_MODIFY); err != nil { + unix.Close(fd) return 0, 0, fmt.Errorf("failed to add inotify watch for %q: %w", evpath, err) } @@ -736,41 +701,6 @@ func (c *Manager) EventChan() (<-chan Event, <-chan error) { return ec, errCh } -func parseMemoryEvents(out map[string]interface{}) (Event, error) { - e := Event{} - if v, ok := out["high"]; ok { - e.High, ok = v.(uint64) - if !ok { - return Event{}, fmt.Errorf("cannot convert high to uint64: %+v", v) - } - } - if v, ok := out["low"]; ok { - e.Low, ok = v.(uint64) - if !ok { - return Event{}, fmt.Errorf("cannot convert low to uint64: %+v", v) - } - } - if v, ok := out["max"]; ok { - e.Max, ok = v.(uint64) - if !ok { - return Event{}, fmt.Errorf("cannot convert max to uint64: %+v", v) - } - } - if v, ok := out["oom"]; ok { - e.OOM, ok = v.(uint64) - if !ok { - return Event{}, fmt.Errorf("cannot convert oom to uint64: %+v", v) - } - } - if v, ok := out["oom_kill"]; ok { - e.OOMKill, ok = v.(uint64) - if !ok { - return Event{}, fmt.Errorf("cannot convert oom_kill to uint64: %+v", v) - } - } - return e, nil -} - func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { defer close(errCh) @@ -779,17 +709,17 @@ func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { errCh <- err return } - defer syscall.Close(fd) + defer unix.Close(fd) for { - buffer := make([]byte, syscall.SizeofInotifyEvent*10) - bytesRead, err := syscall.Read(fd, buffer) + buffer := make([]byte, unix.SizeofInotifyEvent*10) + bytesRead, err := unix.Read(fd, buffer) if err != nil { errCh <- err return } - if bytesRead >= syscall.SizeofInotifyEvent { - out := make(map[string]interface{}) + if bytesRead >= unix.SizeofInotifyEvent { + out := make(map[string]uint64) if err := readKVStatsFile(c.path, "memory.events", out); err != nil { // When cgroup is deleted read may return -ENODEV instead of -ENOENT from open. if _, statErr := os.Lstat(filepath.Join(c.path, "memory.events")); !os.IsNotExist(statErr) { @@ -797,12 +727,13 @@ func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { } return } - e, err := parseMemoryEvents(out) - if err != nil { - errCh <- err - return + ec <- Event{ + Low: out["low"], + High: out["high"], + Max: out["max"], + OOM: out["oom"], + OOMKill: out["oom_kill"], } - ec <- e if c.isCgroupEmpty() { return } @@ -818,7 +749,7 @@ func setDevices(path string, devices []specs.LinuxDeviceCgroup) error { if err != nil { return err } - dirFD, err := unix.Open(path, unix.O_DIRECTORY|unix.O_RDONLY|unix.O_CLOEXEC, 0600) + dirFD, err := unix.Open(path, unix.O_DIRECTORY|unix.O_RDONLY|unix.O_CLOEXEC, 0o600) if err != nil { return fmt.Errorf("cannot get dir FD for %s", path) } diff --git a/vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go b/vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go index 7765018308ab0..f5302444a7f14 100644 --- a/vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go +++ b/vendor/github.com/containerd/cgroups/v3/cgroup2/utils.go @@ -18,6 +18,7 @@ package cgroup2 import ( "bufio" + "errors" "fmt" "io" "math" @@ -25,6 +26,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "unsafe" @@ -39,7 +41,7 @@ import ( const ( cgroupProcs = "cgroup.procs" cgroupThreads = "cgroup.threads" - defaultDirPerm = 0755 + defaultDirPerm = 0o755 ) // defaultFilePerm is a var so that the test framework can change the filemode @@ -92,19 +94,13 @@ func parseCgroupProcsFile(path string) ([]uint64, error) { return out, nil } -func parseKV(raw string) (string, interface{}, error) { +func parseKV(raw string) (string, uint64, error) { parts := strings.Fields(raw) - switch len(parts) { - case 2: - v, err := parseUint(parts[1], 10, 64) - if err != nil { - // if we cannot parse as a uint, parse as a string - return parts[0], parts[1], nil - } - return parts[0], v, nil - default: + if len(parts) != 2 { return "", 0, ErrInvalidFormat } + v, err := parseUint(parts[1], 10, 64) + return parts[0], v, err } func parseUint(s string, base, bitSize int) (uint64, error) { @@ -136,9 +132,7 @@ func parseCgroupFile(path string) (string, error) { } func parseCgroupFromReader(r io.Reader) (string, error) { - var ( - s = bufio.NewScanner(r) - ) + s := bufio.NewScanner(r) for s.Scan() { var ( text = s.Text() @@ -244,18 +238,28 @@ func ToResources(spec *specs.LinuxResources) *Resources { // Gets uint64 parsed content of single value cgroup stat file func getStatFileContentUint64(filePath string) uint64 { - contents, err := os.ReadFile(filePath) + f, err := os.Open(filePath) if err != nil { return 0 } - trimmed := strings.TrimSpace(string(contents)) + defer f.Close() + + // We expect an unsigned 64 bit integer, or a "max" string + // in some cases. + buf := make([]byte, 32) + n, err := f.Read(buf) + if err != nil { + return 0 + } + + trimmed := strings.TrimSpace(string(buf[:n])) if trimmed == "max" { return math.MaxUint64 } res, err := parseUint(trimmed, 10, 64) if err != nil { - logrus.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), filePath) + logrus.Errorf("unable to parse %q as a uint from Cgroup file %q", trimmed, filePath) return res } @@ -385,56 +389,94 @@ func systemdUnitFromPath(path string) string { } func readHugeTlbStats(path string) []*stats.HugeTlbStat { - var usage = []*stats.HugeTlbStat{} - var keyUsage = make(map[string]*stats.HugeTlbStat) - f, err := os.Open(path) - if err != nil { - return usage - } - files, err := f.Readdir(-1) - f.Close() - if err != nil { - return usage + hpSizes := hugePageSizes() + usage := make([]*stats.HugeTlbStat, len(hpSizes)) + for idx, pagesize := range hpSizes { + usage[idx] = &stats.HugeTlbStat{ + Max: getStatFileContentUint64(filepath.Join(path, "hugetlb."+pagesize+".max")), + Current: getStatFileContentUint64(filepath.Join(path, "hugetlb."+pagesize+".current")), + Pagesize: pagesize, + } } + return usage +} - for _, file := range files { - if strings.Contains(file.Name(), "hugetlb") && - (strings.HasSuffix(file.Name(), "max") || strings.HasSuffix(file.Name(), "current")) { - var hugeTlb *stats.HugeTlbStat - var ok bool - fileName := strings.Split(file.Name(), ".") - pageSize := fileName[1] - if hugeTlb, ok = keyUsage[pageSize]; !ok { - hugeTlb = &stats.HugeTlbStat{} - } - hugeTlb.Pagesize = pageSize - out, err := os.ReadFile(filepath.Join(path, file.Name())) - if err != nil { - continue - } - var value uint64 - stringVal := strings.TrimSpace(string(out)) - if stringVal == "max" { - value = math.MaxUint64 - } else { - value, err = strconv.ParseUint(stringVal, 10, 64) - } - if err != nil { - continue +var ( + hPageSizes []string + initHPSOnce sync.Once +) + +// The following idea and implementation is taken pretty much line for line from +// runc. Because the hugetlb files are well known, and the only variable thrown in +// the mix is what huge page sizes you have on your host, this lends itself well +// to doing the work to find the files present once, and then re-using this. This +// saves a os.Readdirnames(0) call to search for hugeltb files on every `manager.Stat` +// call. +// https://github.com/opencontainers/runc/blob/3a2c0c2565644d8a7e0f1dd594a060b21fa96cf1/libcontainer/cgroups/utils.go#L301 +func hugePageSizes() []string { + initHPSOnce.Do(func() { + dir, err := os.OpenFile("/sys/kernel/mm/hugepages", unix.O_DIRECTORY|unix.O_RDONLY, 0) + if err != nil { + return + } + files, err := dir.Readdirnames(0) + dir.Close() + if err != nil { + return + } + + hPageSizes, err = getHugePageSizeFromFilenames(files) + if err != nil { + logrus.Warnf("hugePageSizes: %s", err) + } + }) + + return hPageSizes +} + +func getHugePageSizeFromFilenames(fileNames []string) ([]string, error) { + pageSizes := make([]string, 0, len(fileNames)) + var warn error + + for _, file := range fileNames { + // example: hugepages-1048576kB + val := strings.TrimPrefix(file, "hugepages-") + if len(val) == len(file) { + // Unexpected file name: no prefix found, ignore it. + continue + } + // In all known versions of Linux up to 6.3 the suffix is always + // "kB". If we find something else, produce an error but keep going. + eLen := len(val) - 2 + val = strings.TrimSuffix(val, "kB") + if len(val) != eLen { + // Highly unlikely. + if warn == nil { + warn = errors.New(file + `: invalid suffix (expected "kB")`) } - switch fileName[2] { - case "max": - hugeTlb.Max = value - case "current": - hugeTlb.Current = value + continue + } + size, err := strconv.Atoi(val) + if err != nil { + // Highly unlikely. + if warn == nil { + warn = fmt.Errorf("%s: %w", file, err) } - keyUsage[pageSize] = hugeTlb + continue } + // Model after https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/mm/hugetlb_cgroup.c?id=eff48ddeab782e35e58ccc8853f7386bbae9dec4#n574 + // but in our case the size is in KB already. + if size >= (1 << 20) { + val = strconv.Itoa(size>>20) + "GB" + } else if size >= (1 << 10) { + val = strconv.Itoa(size>>10) + "MB" + } else { + val += "KB" + } + pageSizes = append(pageSizes, val) } - for _, entry := range keyUsage { - usage = append(usage, entry) - } - return usage + + return pageSizes, warn } func getSubreaper() (int, error) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 4ae0e00ece0cc..3483c56266500 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -206,8 +206,8 @@ github.com/container-storage-interface/spec/lib/go/csi # github.com/containerd/cgroups v1.0.4 ## explicit; go 1.17 github.com/containerd/cgroups/stats/v1 -# github.com/containerd/cgroups/v3 v3.0.1 -## explicit; go 1.17 +# github.com/containerd/cgroups/v3 v3.0.2 +## explicit; go 1.18 github.com/containerd/cgroups/v3 github.com/containerd/cgroups/v3/cgroup1 github.com/containerd/cgroups/v3/cgroup1/stats From 649bb2b9b8748a45ac91519b56c4e797d63272b9 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 1 Jul 2023 16:34:46 +0200 Subject: [PATCH 119/293] api: remove "ClusterStore" and "ClusterAdvertise" fields The `ClusterStore` and `ClusterAdvertise` fields were deprecated in commit 616e64b42ffb7ce609d53ad2c7b80e5362e8b9a8 (and would no longer be included in the `/info` API response), and were fully removed in 24.0.0 through commit 68bf777eced495df352be219da9c9c1070af5c99 This patch removes the fields from the swagger file. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 3c905d0db938bfd6d0d5608d0fb6bcab60901b23) Signed-off-by: Sebastiaan van Stijn --- api/swagger.yaml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index c2943888d7528..98da60c89e83a 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -5168,36 +5168,6 @@ definitions: > `swarm/1.2.8`. type: "string" example: "17.06.0-ce" - ClusterStore: - description: | - URL of the distributed storage backend. - - - The storage backend is used for multihost networking (to store - network and endpoint information) and by the node discovery mechanism. - -


- - > **Deprecated**: This field is only propagated when using standalone Swarm - > mode, and overlay networking using an external k/v store. Overlay - > networks with Swarm mode enabled use the built-in raft store, and - > this field will be empty. - type: "string" - example: "consul://consul.corp.example.com:8600/some/path" - ClusterAdvertise: - description: | - The network endpoint that the Engine advertises for the purpose of - node discovery. ClusterAdvertise is a `host:port` combination on which - the daemon is reachable by other hosts. - -


- - > **Deprecated**: This field is only propagated when using standalone Swarm - > mode, and overlay networking using an external k/v store. Overlay - > networks with Swarm mode enabled use the built-in raft store, and - > this field will be empty. - type: "string" - example: "node5.corp.example.com:8000" Runtimes: description: | List of [OCI compliant](https://github.com/opencontainers/runtime-spec) From 7a9c831e6a537c7ac3aea4965cb89e32deb3b67d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 1 Jul 2023 16:36:52 +0200 Subject: [PATCH 120/293] docs: api v1.43: remove "ClusterStore" and "ClusterAdvertise" fields The `ClusterStore` and `ClusterAdvertise` fields were deprecated in commit 616e64b42ffb7ce609d53ad2c7b80e5362e8b9a8 (and would no longer be included in the `/info` API response), and were fully removed in 24.0.0 through commit 68bf777eced495df352be219da9c9c1070af5c99 This patch removes the fields from the swagger file. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit e58a60902c4977d6fa79d7b46ea998123b7e3134) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.43.yaml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/docs/api/v1.43.yaml b/docs/api/v1.43.yaml index c2943888d7528..98da60c89e83a 100644 --- a/docs/api/v1.43.yaml +++ b/docs/api/v1.43.yaml @@ -5168,36 +5168,6 @@ definitions: > `swarm/1.2.8`. type: "string" example: "17.06.0-ce" - ClusterStore: - description: | - URL of the distributed storage backend. - - - The storage backend is used for multihost networking (to store - network and endpoint information) and by the node discovery mechanism. - -


- - > **Deprecated**: This field is only propagated when using standalone Swarm - > mode, and overlay networking using an external k/v store. Overlay - > networks with Swarm mode enabled use the built-in raft store, and - > this field will be empty. - type: "string" - example: "consul://consul.corp.example.com:8600/some/path" - ClusterAdvertise: - description: | - The network endpoint that the Engine advertises for the purpose of - node discovery. ClusterAdvertise is a `host:port` combination on which - the daemon is reachable by other hosts. - -


- - > **Deprecated**: This field is only propagated when using standalone Swarm - > mode, and overlay networking using an external k/v store. Overlay - > networks with Swarm mode enabled use the built-in raft store, and - > this field will be empty. - type: "string" - example: "node5.corp.example.com:8000" Runtimes: description: | List of [OCI compliant](https://github.com/opencontainers/runtime-spec) From af25852baa2d5e11cabf0d91b129580e47de8f96 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 1 Jul 2023 16:37:51 +0200 Subject: [PATCH 121/293] docs: api v1.42: remove "ClusterStore" and "ClusterAdvertise" fields The `ClusterStore` and `ClusterAdvertise` fields were deprecated in commit 616e64b42ffb7ce609d53ad2c7b80e5362e8b9a8 (and would no longer be included in the `/info` API response), and were fully removed in 24.0.0 through commit 68bf777eced495df352be219da9c9c1070af5c99 This patch removes the fields from the swagger file. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit e8f206972adeb38b12aa2f4c0f784886f93779f4) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.42.yaml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/docs/api/v1.42.yaml b/docs/api/v1.42.yaml index d8d846cb318fb..7b3dbd65b33fe 100644 --- a/docs/api/v1.42.yaml +++ b/docs/api/v1.42.yaml @@ -5136,36 +5136,6 @@ definitions: > `swarm/1.2.8`. type: "string" example: "17.06.0-ce" - ClusterStore: - description: | - URL of the distributed storage backend. - - - The storage backend is used for multihost networking (to store - network and endpoint information) and by the node discovery mechanism. - -


- - > **Deprecated**: This field is only propagated when using standalone Swarm - > mode, and overlay networking using an external k/v store. Overlay - > networks with Swarm mode enabled use the built-in raft store, and - > this field will be empty. - type: "string" - example: "consul://consul.corp.example.com:8600/some/path" - ClusterAdvertise: - description: | - The network endpoint that the Engine advertises for the purpose of - node discovery. ClusterAdvertise is a `host:port` combination on which - the daemon is reachable by other hosts. - -


- - > **Deprecated**: This field is only propagated when using standalone Swarm - > mode, and overlay networking using an external k/v store. Overlay - > networks with Swarm mode enabled use the built-in raft store, and - > this field will be empty. - type: "string" - example: "node5.corp.example.com:8000" Runtimes: description: | List of [OCI compliant](https://github.com/opencontainers/runtime-spec) From 0ef846ce2e1f297678c5e1f1f7cc74ed8f91ba9a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 12:25:57 +0200 Subject: [PATCH 122/293] api: remove outdated information from ServerVersion This field's documentation was still referring to the Swarm V1 API, which is deprecated, and the link redirects to SwarmKit. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 92f1ddaf0a4cc3e77caaf43ef20b01aeebba6a15) Signed-off-by: Sebastiaan van Stijn --- api/swagger.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index 98da60c89e83a..aefbcb8e87cc9 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -5162,12 +5162,8 @@ definitions: ServerVersion: description: | Version string of the daemon. - - > **Note**: the [standalone Swarm API](https://docs.docker.com/swarm/swarm-api/) - > returns the Swarm version instead of the daemon version, for example - > `swarm/1.2.8`. type: "string" - example: "17.06.0-ce" + example: "24.0.2" Runtimes: description: | List of [OCI compliant](https://github.com/opencontainers/runtime-spec) From 335ed29345642bbe4fc9f05f9a8883dee68d6970 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 12:32:36 +0200 Subject: [PATCH 123/293] docs: api v1.43: remove outdated information from ServerVersion This field's documentation was still referring to the Swarm V1 API, which is deprecated, and the link redirects to SwarmKit. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 18d77ff455c398c1a0d6e904b042d64b27ba02cd) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.43.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/api/v1.43.yaml b/docs/api/v1.43.yaml index 98da60c89e83a..aefbcb8e87cc9 100644 --- a/docs/api/v1.43.yaml +++ b/docs/api/v1.43.yaml @@ -5162,12 +5162,8 @@ definitions: ServerVersion: description: | Version string of the daemon. - - > **Note**: the [standalone Swarm API](https://docs.docker.com/swarm/swarm-api/) - > returns the Swarm version instead of the daemon version, for example - > `swarm/1.2.8`. type: "string" - example: "17.06.0-ce" + example: "24.0.2" Runtimes: description: | List of [OCI compliant](https://github.com/opencontainers/runtime-spec) From acb95e4544e925719e4c2441ec01c9c6120d1f3e Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 12:32:48 +0200 Subject: [PATCH 124/293] docs: api v1.42: remove outdated information from ServerVersion This field's documentation was still referring to the Swarm V1 API, which is deprecated, and the link redirects to SwarmKit. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit b770a50dee5f7b30c40938a26d10d0aa43f67fe2) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.42.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/api/v1.42.yaml b/docs/api/v1.42.yaml index 7b3dbd65b33fe..dadb2f1fb8542 100644 --- a/docs/api/v1.42.yaml +++ b/docs/api/v1.42.yaml @@ -5130,12 +5130,8 @@ definitions: ServerVersion: description: | Version string of the daemon. - - > **Note**: the [standalone Swarm API](https://docs.docker.com/swarm/swarm-api/) - > returns the Swarm version instead of the daemon version, for example - > `swarm/1.2.8`. type: "string" - example: "17.06.0-ce" + example: "23.0.0" Runtimes: description: | List of [OCI compliant](https://github.com/opencontainers/runtime-spec) From f66ef3160547afeeb21c0065f21074e5c43459bd Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 12:32:57 +0200 Subject: [PATCH 125/293] docs: api v1.41: remove outdated information from ServerVersion This field's documentation was still referring to the Swarm V1 API, which is deprecated, and the link redirects to SwarmKit. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ed0dbb8518f3bec8e193f3911af43236c47f8b46) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.41.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/api/v1.41.yaml b/docs/api/v1.41.yaml index 0b9e655900210..64ccc59c7d5fd 100644 --- a/docs/api/v1.41.yaml +++ b/docs/api/v1.41.yaml @@ -5100,12 +5100,8 @@ definitions: ServerVersion: description: | Version string of the daemon. - - > **Note**: the [standalone Swarm API](https://docs.docker.com/swarm/swarm-api/) - > returns the Swarm version instead of the daemon version, for example - > `swarm/1.2.8`. type: "string" - example: "17.06.0-ce" + example: "20.10.25" ClusterStore: description: | URL of the distributed storage backend. From 32bcbdfe657c8bd37c6ee88ac76a47533495442e Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Mon, 1 Aug 2022 16:06:08 -0400 Subject: [PATCH 126/293] api: swagger: add missing "force" query arg on plugin disable This has been around for a long time - since v17.04 (API v1.28) but was never documented. It allows removing a plugin even if it's still in use. Signed-off-by: Milas Bowman (cherry picked from commit eb0edeafddb65794dbd958267d89a2f7981b02c8) Signed-off-by: Sebastiaan van Stijn --- api/swagger.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/swagger.yaml b/api/swagger.yaml index 98da60c89e83a..00b8690d5b234 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -10363,6 +10363,12 @@ paths: default if omitted. required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: From 50fb65f0f534d85c9e8aed1ee3fefadfbc1d5eda Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 12:50:28 +0200 Subject: [PATCH 127/293] docs: api: amend changelog for API 1.28 for "force" option This option was added in 8cb2229cd18c53bdbf36301f26db565a50027d6a for API version 1.28, but forgot to update the documentation and version history. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit aba8e04ab1e6536bf51fd27f1941d2410b04d028) Signed-off-by: Sebastiaan van Stijn --- docs/api/version-history.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api/version-history.md b/docs/api/version-history.md index f5f20f503d288..da04e3f2d3f41 100644 --- a/docs/api/version-history.md +++ b/docs/api/version-history.md @@ -475,6 +475,7 @@ keywords: "API, Docker, rcli, REST, documentation" * `POST /services/create` and `POST /services/(id or name)/update` now accept an optional `RollbackConfig` object which specifies rollback options. * `GET /services` now supports a `mode` filter to filter services based on the service mode (either `global` or `replicated`). * `POST /containers/(name)/update` now supports updating `NanoCpus` that represents CPU quota in units of 10-9 CPUs. +* `POST /plugins/{name}/disable` now accepts a `force` query-parameter to disable a plugin even if still in use. ## v1.27 API changes From b732cfd3928e5efb4bdd36154503a9d1a1f42c69 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 13:02:32 +0200 Subject: [PATCH 128/293] docs: api v1.43: add missing "force" query arg on plugin disable This option was added in 8cb2229cd18c53bdbf36301f26db565a50027d6a for API version 1.28, but forgot to update the documentation and version history. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 85ccb25eb8ef72837e50330acf390d7b44157e8b) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.43.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/api/v1.43.yaml b/docs/api/v1.43.yaml index 98da60c89e83a..00b8690d5b234 100644 --- a/docs/api/v1.43.yaml +++ b/docs/api/v1.43.yaml @@ -10363,6 +10363,12 @@ paths: default if omitted. required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: From b5aacf8161c99dcd6e04304b14846db7c5094951 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 13:02:45 +0200 Subject: [PATCH 129/293] docs: api v1.42: add missing "force" query arg on plugin disable This option was added in 8cb2229cd18c53bdbf36301f26db565a50027d6a for API version 1.28, but forgot to update the documentation and version history. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit a4bdfb963fbb6f8d3f86c4731033362224b5c083) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.42.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/api/v1.42.yaml b/docs/api/v1.42.yaml index 7b3dbd65b33fe..cb9c2491d3b84 100644 --- a/docs/api/v1.42.yaml +++ b/docs/api/v1.42.yaml @@ -10345,6 +10345,12 @@ paths: default if omitted. required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: From 7adb590e162da0974b32741e499d0973d3c9255d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 13:02:57 +0200 Subject: [PATCH 130/293] docs: api v1.41: add missing "force" query arg on plugin disable This option was added in 8cb2229cd18c53bdbf36301f26db565a50027d6a for API version 1.28, but forgot to update the documentation and version history. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 892e9f2c23ad20009c8d3f4c6b5fdf7ff772f0e4) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.41.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/api/v1.41.yaml b/docs/api/v1.41.yaml index 0b9e655900210..e0545039d3851 100644 --- a/docs/api/v1.41.yaml +++ b/docs/api/v1.41.yaml @@ -9967,6 +9967,12 @@ paths: default if omitted. required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: From 5892aae60fe4ef681720606afed51e67fc2a7f86 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 2 Jul 2023 13:04:06 +0200 Subject: [PATCH 131/293] docs: api v1.28 - v1.40: add missing "force" query arg on plugin disable This option was added in 8cb2229cd18c53bdbf36301f26db565a50027d6a for API version 1.28, but forgot to update the documentation and version history. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit f6258f70cb43b12654e2c185ea0aea234485013d) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.28.yaml | 6 ++++++ docs/api/v1.29.yaml | 6 ++++++ docs/api/v1.30.yaml | 6 ++++++ docs/api/v1.31.yaml | 6 ++++++ docs/api/v1.32.yaml | 6 ++++++ docs/api/v1.33.yaml | 6 ++++++ docs/api/v1.34.yaml | 6 ++++++ docs/api/v1.35.yaml | 6 ++++++ docs/api/v1.36.yaml | 6 ++++++ docs/api/v1.37.yaml | 6 ++++++ docs/api/v1.38.yaml | 6 ++++++ docs/api/v1.39.yaml | 6 ++++++ docs/api/v1.40.yaml | 6 ++++++ 13 files changed, 78 insertions(+) diff --git a/docs/api/v1.28.yaml b/docs/api/v1.28.yaml index 52c2fa4165715..44bb67f025748 100644 --- a/docs/api/v1.28.yaml +++ b/docs/api/v1.28.yaml @@ -6839,6 +6839,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.29.yaml b/docs/api/v1.29.yaml index d675b7c61943e..ed69b68892313 100644 --- a/docs/api/v1.29.yaml +++ b/docs/api/v1.29.yaml @@ -6881,6 +6881,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.30.yaml b/docs/api/v1.30.yaml index e3c8e4280eaf3..535a6205e23f8 100644 --- a/docs/api/v1.30.yaml +++ b/docs/api/v1.30.yaml @@ -7105,6 +7105,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.31.yaml b/docs/api/v1.31.yaml index e38e6eb9aaab3..8535ffe23728a 100644 --- a/docs/api/v1.31.yaml +++ b/docs/api/v1.31.yaml @@ -7203,6 +7203,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.32.yaml b/docs/api/v1.32.yaml index 1dfd274f44e55..b843bc63406c4 100644 --- a/docs/api/v1.32.yaml +++ b/docs/api/v1.32.yaml @@ -8248,6 +8248,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.33.yaml b/docs/api/v1.33.yaml index 1d4034fbd0e57..3fa7eb43d5c58 100644 --- a/docs/api/v1.33.yaml +++ b/docs/api/v1.33.yaml @@ -8257,6 +8257,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.34.yaml b/docs/api/v1.34.yaml index 966b090e59d8a..d90d4dbbf84d4 100644 --- a/docs/api/v1.34.yaml +++ b/docs/api/v1.34.yaml @@ -8298,6 +8298,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.35.yaml b/docs/api/v1.35.yaml index 2591fb6a2e7df..a3ed1494669fe 100644 --- a/docs/api/v1.35.yaml +++ b/docs/api/v1.35.yaml @@ -8310,6 +8310,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.36.yaml b/docs/api/v1.36.yaml index aa808e48b4e8a..d5f4da5440eb6 100644 --- a/docs/api/v1.36.yaml +++ b/docs/api/v1.36.yaml @@ -8354,6 +8354,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.37.yaml b/docs/api/v1.37.yaml index 79496b06ef44f..b9290e1fea1dc 100644 --- a/docs/api/v1.37.yaml +++ b/docs/api/v1.37.yaml @@ -8397,6 +8397,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.38.yaml b/docs/api/v1.38.yaml index 4d1a28e548659..af7b11817d86e 100644 --- a/docs/api/v1.38.yaml +++ b/docs/api/v1.38.yaml @@ -8458,6 +8458,12 @@ paths: description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.39.yaml b/docs/api/v1.39.yaml index 5e565b4dc12a8..b7d7943af5b79 100644 --- a/docs/api/v1.39.yaml +++ b/docs/api/v1.39.yaml @@ -9445,6 +9445,12 @@ paths: default if omitted. required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: diff --git a/docs/api/v1.40.yaml b/docs/api/v1.40.yaml index 818f849390544..98df0853939e9 100644 --- a/docs/api/v1.40.yaml +++ b/docs/api/v1.40.yaml @@ -9760,6 +9760,12 @@ paths: default if omitted. required: true type: "string" + - name: "force" + in: "query" + description: | + Force disable a plugin even if still in use. + required: false + type: "boolean" tags: ["Plugin"] /plugins/{name}/upgrade: post: From d3893b58ff9dc2fd1f25e41e6e6766694632787e Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 7 Jul 2023 14:54:04 +0200 Subject: [PATCH 132/293] daemon: daemon.prepareMountPoints(): fix panic if mount is not a volume The daemon.lazyInitializeVolume() function only handles restoring Volumes if a Driver is specified. The Container's MountPoints field may also contain other kind of mounts (e.g., bind-mounts). Those were ignored, and don't return an error; https://github.com/moby/moby/blob/1d9c8619cded4657af1529779c5771127e8ad0e7/daemon/volumes.go#L243-L252C2 However, the prepareMountPoints() assumed each MountPoint was a volume, and logged an informational message about the volume being restored; https://github.com/moby/moby/blob/1d9c8619cded4657af1529779c5771127e8ad0e7/daemon/mounts.go#L18-L25 This would panic if the MountPoint was not a volume; github.com/docker/docker/daemon.(*Daemon).prepareMountPoints(0xc00054b7b8?, 0xc0007c2500) /root/rpmbuild/BUILD/src/engine/.gopath/src/github.com/docker/docker/daemon/mounts.go:24 +0x1c0 github.com/docker/docker/daemon.(*Daemon).restore.func5(0xc0007c2500, 0x0?) /root/rpmbuild/BUILD/src/engine/.gopath/src/github.com/docker/docker/daemon/daemon.go:552 +0x271 created by github.com/docker/docker/daemon.(*Daemon).restore /root/rpmbuild/BUILD/src/engine/.gopath/src/github.com/docker/docker/daemon/daemon.go:530 +0x8d8 panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x30 pc=0x564e9be4c7c0] This issue was introduced in 647c2a6cdd86d79230df1bf690d0b6a2930d6db2 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit a490248f4d19164d78d3ef4f91cf142c3aad1790) Signed-off-by: Sebastiaan van Stijn --- daemon/mounts.go | 4 ++++ integration/daemon/daemon_test.go | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/daemon/mounts.go b/daemon/mounts.go index ad637df03d073..6a4d22cc091d4 100644 --- a/daemon/mounts.go +++ b/daemon/mounts.go @@ -18,6 +18,10 @@ func (daemon *Daemon) prepareMountPoints(container *container.Container) error { if err := daemon.lazyInitializeVolume(container.ID, config); err != nil { return err } + if config.Volume == nil { + // FIXME(thaJeztah): should we check for config.Type here as well? (i.e., skip bind-mounts etc) + continue + } if alive { log.G(context.TODO()).WithFields(logrus.Fields{ "container": container.ID, diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index 9dcaed374757b..b6e00c81b9c04 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -436,6 +436,24 @@ func testLiveRestoreVolumeReferences(t *testing.T) { err = c.VolumeRemove(ctx, v.Name, false) assert.NilError(t, err) }) + + // Make sure that we don't panic if the container has bind-mounts + // (which should not be "restored") + // Regression test for https://github.com/moby/moby/issues/45898 + t.Run("container with bind-mounts", func(t *testing.T) { + m := mount.Mount{ + Type: mount.TypeBind, + Source: os.TempDir(), + Target: "/foo", + } + cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("top")) + defer c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) + + d.Restart(t, "--live-restore", "--iptables=false") + + err := c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) + assert.NilError(t, err) + }) } func TestDaemonDefaultBridgeWithFixedCidrButNoBip(t *testing.T) { From a3049653c17f1e4b1c319272413fb3306cbd7848 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Wed, 5 Jul 2023 12:09:37 -0400 Subject: [PATCH 133/293] pkg/plugins: make unit test less time sensitive TestClientWithRequestTimeout has been observed to flake in CI. The timing in the test is quite tight, only giving the client a 10ms window to time out, which could potentially be missed if the host is under load and the goroutine scheduling is unlucky. Give the client a full five seconds of grace to time out before failing the test. Signed-off-by: Cory Snider (cherry picked from commit 9cee34bc94fb5c78a8a79a0b36118d13f27f2f8b) Signed-off-by: Cory Snider --- pkg/plugins/client_test.go | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/pkg/plugins/client_test.go b/pkg/plugins/client_test.go index f93734d367207..f60d0cdb6b36d 100644 --- a/pkg/plugins/client_test.go +++ b/pkg/plugins/client_test.go @@ -3,6 +3,7 @@ package plugins // import "github.com/docker/docker/pkg/plugins" import ( "bytes" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" @@ -13,7 +14,6 @@ import ( "github.com/docker/docker/pkg/plugins/transport" "github.com/docker/go-connections/tlsconfig" - "github.com/pkg/errors" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" ) @@ -240,22 +240,39 @@ func TestClientWithRequestTimeout(t *testing.T) { Timeout() bool } - timeout := 1 * time.Millisecond + unblock := make(chan struct{}) testHandler := func(w http.ResponseWriter, r *http.Request) { - time.Sleep(timeout + 10*time.Millisecond) + select { + case <-unblock: + case <-r.Context().Done(): + } w.WriteHeader(http.StatusOK) } srv := httptest.NewServer(http.HandlerFunc(testHandler)) - defer srv.Close() + defer func() { + close(unblock) + srv.Close() + }() client := &Client{http: srv.Client(), requestFactory: &testRequestWrapper{srv}} - _, err := client.callWithRetry("/Plugin.Hello", nil, false, WithRequestTimeout(timeout)) - assert.Assert(t, is.ErrorContains(err, ""), "expected error") - - var tErr timeoutError - assert.Assert(t, errors.As(err, &tErr)) - assert.Assert(t, tErr.Timeout()) + errCh := make(chan error, 1) + go func() { + _, err := client.callWithRetry("/Plugin.Hello", nil, false, WithRequestTimeout(time.Millisecond)) + errCh <- err + }() + + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + select { + case err := <-errCh: + var tErr timeoutError + if assert.Check(t, errors.As(err, &tErr), "want timeout error, got %T", err) { + assert.Check(t, tErr.Timeout()) + } + case <-timer.C: + t.Fatal("client request did not time out in time") + } } type testRequestWrapper struct { From 0e88c57c470651400d677544b2681d44b8b14525 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Wed, 5 Jul 2023 13:49:53 -0400 Subject: [PATCH 134/293] integration: disable iptables in parallel tests Multiple daemons starting/running concurrently can collide with each other when editing iptables rules. Most integration tests which opt into parallelism and start daemons work around this problem by starting the daemon with the --iptables=false option. However, some of the tests neglect to pass the option when starting or restarting the daemon, resulting in those tests being flaky. Audit the integration tests which call t.Parallel() and (*Daemon).Stop() and add --iptables=false arguments where needed. Signed-off-by: Cory Snider (cherry picked from commit cdcb7c28c5f6d29652fa9d37dc45041b190d1cd4) Signed-off-by: Cory Snider --- integration/container/daemon_linux_test.go | 8 ++++---- integration/container/daemon_test.go | 2 +- integration/image/import_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/container/daemon_linux_test.go b/integration/container/daemon_linux_test.go index d1d6c61a9df5d..b67dece2c9b2c 100644 --- a/integration/container/daemon_linux_test.go +++ b/integration/container/daemon_linux_test.go @@ -140,7 +140,7 @@ func TestDaemonHostGatewayIP(t *testing.T) { // Verify the IP in /etc/hosts is same as host-gateway-ip d := daemon.New(t) // Verify the IP in /etc/hosts is same as the default bridge's IP - d.StartWithBusybox(t) + d.StartWithBusybox(t, "--iptables=false") c := d.NewClientT(t) ctx := context.Background() cID := container.Run(ctx, t, c, @@ -157,7 +157,7 @@ func TestDaemonHostGatewayIP(t *testing.T) { d.Stop(t) // Verify the IP in /etc/hosts is same as host-gateway-ip - d.StartWithBusybox(t, "--host-gateway-ip=6.7.8.9") + d.StartWithBusybox(t, "--iptables=false", "--host-gateway-ip=6.7.8.9") cID = container.Run(ctx, t, c, container.WithExtraHost("host.docker.internal:host-gateway"), ) @@ -208,7 +208,7 @@ func TestRestartDaemonWithRestartingContainer(t *testing.T) { c.HasBeenStartedBefore = true }) - d.Start(t) + d.Start(t, "--iptables=false") ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -256,7 +256,7 @@ func TestHardRestartWhenContainerIsRunning(t *testing.T) { }) } - d.Start(t) + d.Start(t, "--iptables=false") t.Run("RestartPolicy=none", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) diff --git a/integration/container/daemon_test.go b/integration/container/daemon_test.go index 94468a440914f..a0a66b72471de 100644 --- a/integration/container/daemon_test.go +++ b/integration/container/daemon_test.go @@ -43,7 +43,7 @@ func TestContainerKillOnDaemonStart(t *testing.T) { assert.Assert(t, inspect.State.Running) assert.NilError(t, d.Kill()) - d.Start(t) + d.Start(t, "--iptables=false") inspect, err = client.ContainerInspect(ctx, id) assert.Check(t, is.Nil(err)) diff --git a/integration/image/import_test.go b/integration/image/import_test.go index 110ab87a5faef..9ee647867a6c7 100644 --- a/integration/image/import_test.go +++ b/integration/image/import_test.go @@ -27,7 +27,7 @@ func TestImportExtremelyLargeImageWorks(t *testing.T) { // Spin up a new daemon, so that we can run this test in parallel (it's a slow test) d := daemon.New(t) - d.Start(t) + d.Start(t, "--iptables=false") defer d.Stop(t) client := d.NewClientT(t) From 6c7f6c2d475f69a4eb19a5e7756e4e765df7143c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 10 Jul 2023 10:57:54 +0200 Subject: [PATCH 135/293] daemon/containerd: fix assignment to entry in nil map during commit A panic would happen when converting an config that had ports exposed, because the ExposedPorts map in the OCI-spec was not initialized. This could happen when committing a container, or when using the classic builder and the parent image had ports exposed, for example FROM busybox AS stage0 EXPOSE 80 FROM stage0 AS stage1 RUN echo hello Example of the panic: 2023/07/07 15:13:02 http: panic serving @: assignment to entry in nil map goroutine 1944 [running]: net/http.(*conn).serve.func1() /usr/local/go/src/net/http/server.go:1854 +0xbf panic({0x45f660, 0xb6a8d0}) /usr/local/go/src/runtime/panic.go:890 +0x263 github.com/docker/docker/daemon/containerd.containerConfigToOciImageConfig(...) /go/src/github.com/docker/docker/daemon/containerd/image_import.go:397 github.com/docker/docker/daemon/containerd.generateCommitImageConfig({0xc001470498, {0x0, 0x0}, {0xc000c437d8, 0x5}, {0x0, 0x0}, {0xc000c43b27, 0x5}, {0x0, ...}, ...}, ...) /go/src/github.com/docker/docker/daemon/containerd/image_commit.go:138 +0x40e github.com/docker/docker/daemon/containerd.(*ImageService).CommitImage(0xc0008853e0, {0xb8f660, 0xc000c4f7c0}, {{0x0, 0x0}, {0x0, 0x0}, 0xc00104b900, 0xc00104b180, {0xc0011a7640, ...}, ...}) /go/src/github.com/docker/docker/daemon/containerd/image_commit.go:82 +0x73b github.com/docker/docker/daemon/containerd.(*ImageService).CommitBuildStep(0xc0008853e0, {0xb8f660, 0xc000c4f7c0}, {{0x0, 0x0}, {0x0, 0x0}, 0xc00104b900, 0xc00104b180, {0xc0011a7640, ...}, ...}) /go/src/github.com/docker/docker/daemon/containerd/image_commit.go:308 +0x110 github.com/docker/docker/builder/dockerfile.(*Builder).commitContainer(0xc0012b8cc0, {0xb8f660, 0xc000c4f7c0}, 0xc0010b2b60, {0xc0011a7640, 0x40}, 0xc00104b180) /go/src/github.com/docker/docker/builder/dockerfile/internals.go:61 +0x168 github.com/docker/docker/builder/dockerfile.(*Builder).commit(0xc0012b8cc0, {0xb8f660, 0xc000c4f7c0}, 0xc0010b2b60, {0xc0012a7d80?, 0xc001340060?}) /go/src/github.com/docker/docker/builder/dockerfile/internals.go:45 +0x1aa github.com/docker/docker/builder/dockerfile.dispatchLabel({0xb8f660, 0xc000c4f7c0}, {0xc0010b2b60, 0xc000c6b628, 0xc0012b8cc0, {0xb80f60, 0xc0011a46c0}, 0xc000bc2560}, 0x1e24a85?) /go/src/github.com/docker/docker/builder/dockerfile/dispatchers.go:83 +0x258 github.com/docker/docker/builder/dockerfile.dispatch({0xb8f660, 0xc000c4f7c0}, {0xc0010b2b60, 0xc000c6b628, 0xc0012b8cc0, {0xb80f60, 0xc0011a46c0}, 0xc000bc2560}, {0xb7be40, 0xc00111cde0}) /go/src/github.com/docker/docker/builder/dockerfile/evaluator.go:74 +0x529 github.com/docker/docker/builder/dockerfile.(*Builder).dispatchDockerfileWithCancellation(0xc0012b8cc0, {0xb8f660, 0xc000c4f7c0}, {0xc000b1d380, 0x1, 0xc0011a4660?}, {0x0, 0x0, 0x0?}, 0x5c, ...) /go/src/github.com/docker/docker/builder/dockerfile/builder.go:296 +0x8f2 github.com/docker/docker/builder/dockerfile.(*Builder).build(0xc0012b8cc0, {0xb8f660, 0xc000c4f7c0}, {0xb80f60, 0xc0011a46c0}, 0xc0011a49f0) /go/src/github.com/docker/docker/builder/dockerfile/builder.go:211 +0x2e5 github.com/docker/docker/builder/dockerfile.(*BuildManager).Build(0xc0008868c0, {0xb8f708, 0xc0011a44b0}, {{0xb789c0, 0xc0011a4540}, {{0xb6b940, 0xc000c22a50}, {0xb6c5e0, 0xc000c22a68}, {0xb6c5e0, ...}, ...}, ...}) /go/src/github.com/docker/docker/builder/dockerfile/builder.go:98 +0x358 github.com/docker/docker/api/server/backend/build.(*Backend).Build(0xc0007d0870, {0xb8f708, 0xc0011a44b0}, {{0xb789c0, 0xc0011a4540}, {{0xb6b940, 0xc000c22a50}, {0xb6c5e0, 0xc000c22a68}, {0xb6c5e0, ...}, ...}, ...}) /go/src/github.com/docker/docker/api/server/backend/build/backend.go:69 +0x186 github.com/docker/docker/api/server/router/build.(*buildRouter).postBuild(0xc0008333c0, {0xb8f708, 0xc0011a44b0}, {0xb8e130, 0xc0000ed500}, 0xc0010d4800, 0xc0012df760?) /go/src/github.com/docker/docker/api/server/router/build/build_routes.go:280 +0x7a6 github.com/docker/docker/api/server/middleware.ExperimentalMiddleware.WrapHandler.func1({0xb8f708, 0xc0011a44b0}, {0xb8e130?, 0xc0000ed500?}, 0x36cf80?, 0xc0010ab550?) /go/src/github.com/docker/docker/api/server/middleware/experimental.go:26 +0x15b github.com/docker/docker/api/server/middleware.VersionMiddleware.WrapHandler.func1({0xb8f708, 0xc0011a4480}, {0xb8e130, 0xc0000ed500}, 0xc000d787e8?, 0xc000d787a0?) /go/src/github.com/docker/docker/api/server/middleware/version.go:62 +0x4d7 github.com/docker/docker/pkg/authorization.(*Middleware).WrapHandler.func1({0xb8f708, 0xc0011a4480}, {0xb8e130?, 0xc0000ed500?}, 0xc0010d4800, 0xc0010ab500?) /go/src/github.com/docker/docker/pkg/authorization/middleware.go:59 +0x649 github.com/docker/docker/api/server.(*Server).makeHTTPHandler.func1({0xb8e130, 0xc0000ed500}, 0xc0010d4700) /go/src/github.com/docker/docker/api/server/server.go:53 +0x1ce net/http.HandlerFunc.ServeHTTP(0xc0010d4600?, {0xb8e130?, 0xc0000ed500?}, 0xc000d789e8?) /usr/local/go/src/net/http/server.go:2122 +0x2f github.com/docker/docker/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc0001a7e00, {0xb8e130, 0xc0000ed500}, 0xc000d37600) /go/src/github.com/docker/docker/vendor/github.com/gorilla/mux/mux.go:210 +0x1cf net/http.serverHandler.ServeHTTP({0xb7ec58?}, {0xb8e130, 0xc0000ed500}, 0xc000d37600) /usr/local/go/src/net/http/server.go:2936 +0x316 net/http.(*conn).serve(0xc0012661b0, {0xb8f708, 0xc000fd0360}) /usr/local/go/src/net/http/server.go:1995 +0x612 created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:3089 +0x5ed Signed-off-by: Sebastiaan van Stijn (cherry picked from commit a0e1155b2843940fe98a8afd75ee33c3e9431ecf) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/image_import.go | 7 +++++-- daemon/containerd/image_import_test.go | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 daemon/containerd/image_import_test.go diff --git a/daemon/containerd/image_import.go b/daemon/containerd/image_import.go index e4716642df360..081b073eeb4a6 100644 --- a/daemon/containerd/image_import.go +++ b/daemon/containerd/image_import.go @@ -392,8 +392,11 @@ func containerConfigToOciImageConfig(cfg *container.Config) ocispec.ImageConfig StopSignal: cfg.StopSignal, ArgsEscaped: cfg.ArgsEscaped, } - for k, v := range cfg.ExposedPorts { - ociCfg.ExposedPorts[string(k)] = v + if len(cfg.ExposedPorts) > 0 { + ociCfg.ExposedPorts = map[string]struct{}{} + for k, v := range cfg.ExposedPorts { + ociCfg.ExposedPorts[string(k)] = v + } } return ociCfg diff --git a/daemon/containerd/image_import_test.go b/daemon/containerd/image_import_test.go new file mode 100644 index 0000000000000..1b2e20f37125c --- /dev/null +++ b/daemon/containerd/image_import_test.go @@ -0,0 +1,22 @@ +package containerd + +import ( + "testing" + + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "gotest.tools/v3/assert" + is "gotest.tools/v3/assert/cmp" +) + +// regression test for https://github.com/moby/moby/issues/45904 +func TestContainerConfigToOciImageConfig(t *testing.T) { + ociCFG := containerConfigToOciImageConfig(&container.Config{ + ExposedPorts: nat.PortSet{ + "80/tcp": struct{}{}, + }, + }) + + expected := map[string]struct{}{"80/tcp": {}} + assert.Check(t, is.DeepEqual(ociCFG.ExposedPorts, expected)) +} From a4b1a5aef409de523a8948d7527c8c37b3e411d0 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Wed, 12 Jul 2023 19:44:35 +0200 Subject: [PATCH 136/293] vendor: github.com/moby/buildkit@v0.11 0a0807e full diff https://github.com/moby/buildkit/compare/798ad6b...0a15675 Signed-off-by: CrazyMax --- builder/builder-next/worker/worker.go | 2 +- vendor.mod | 2 +- vendor.sum | 4 ++-- vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index ef5cc1716b685..c0715d2564c9f 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -50,7 +50,7 @@ import ( ) func init() { - version.Version = "v0.11.7-0.20230525183624-798ad6b0ce9f" + version.Version = "v0.11.6+0a15675913b7" } const labelCreatedAt = "buildkit/createdat" diff --git a/vendor.mod b/vendor.mod index 04886abac8287..d395f067930b3 100644 --- a/vendor.mod +++ b/vendor.mod @@ -56,7 +56,7 @@ require ( github.com/klauspost/compress v1.16.3 github.com/miekg/dns v1.1.43 github.com/mistifyio/go-zfs/v3 v3.0.1 - github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go + github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go github.com/moby/ipvs v1.1.0 github.com/moby/locker v1.0.1 github.com/moby/patternmatcher v0.5.0 diff --git a/vendor.sum b/vendor.sum index d03c012de7c22..4c9e9c9e08690 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1043,8 +1043,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ= -github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f h1:9wobL03Y6U8azuDLUqYblbUdVU9jpjqecDdW7w4wZtI= -github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f/go.mod h1:GCqKfHhz+pddzfgaR7WmHVEE3nKKZMMDPpK8mh3ZLv4= +github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 h1:9gjbrmALOUAJCqWL4RTwydPUiepMnkc3BNbBFiFiBeU= +github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7/go.mod h1:GCqKfHhz+pddzfgaR7WmHVEE3nKKZMMDPpK8mh3ZLv4= github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= diff --git a/vendor/modules.txt b/vendor/modules.txt index 3483c56266500..769853c517e9a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -576,7 +576,7 @@ github.com/mistifyio/go-zfs/v3 # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 -# github.com/moby/buildkit v0.11.7-0.20230525183624-798ad6b0ce9f +# github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 ## explicit; go 1.18 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types From 9ff2c3918c1ca4be78734894a9f86db963499a5c Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Wed, 12 Jul 2023 16:35:07 +0200 Subject: [PATCH 137/293] ci(buildkit): match moby go version for buildkit tests Signed-off-by: CrazyMax (cherry picked from commit ee9fe2c838075e5be0b57376e7807ac784a9e5d2) --- .github/workflows/buildkit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 8dc35575ca6d4..f0f64a576fcd3 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -13,6 +13,7 @@ on: pull_request: env: + GO_VERSION: "1.20.5" DESTDIR: ./build jobs: From 6c5144d3e5f57386c98da97e5717fc47ef6336b2 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Thu, 13 Jul 2023 13:37:26 +0200 Subject: [PATCH 138/293] Add t.Helper() to the cli test helper functions Signed-off-by: Djordje Lukic --- integration-cli/cli/cli.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration-cli/cli/cli.go b/integration-cli/cli/cli.go index 068a8cfbebfcd..76d2745fa2631 100644 --- a/integration-cli/cli/cli.go +++ b/integration-cli/cli/cli.go @@ -32,22 +32,26 @@ func DockerCmd(t testing.TB, args ...string) *icmd.Result { // BuildCmd executes the specified docker build command and expect a success func BuildCmd(t testing.TB, name string, cmdOperators ...CmdOperator) *icmd.Result { + t.Helper() return Docker(Args("build", "-t", name), cmdOperators...).Assert(t, icmd.Success) } // InspectCmd executes the specified docker inspect command and expect a success func InspectCmd(t testing.TB, name string, cmdOperators ...CmdOperator) *icmd.Result { + t.Helper() return Docker(Args("inspect", name), cmdOperators...).Assert(t, icmd.Success) } // WaitRun will wait for the specified container to be running, maximum 5 seconds. func WaitRun(t testing.TB, name string, cmdOperators ...CmdOperator) { + t.Helper() waitForInspectResult(t, name, "{{.State.Running}}", "true", 5*time.Second, cmdOperators...) } // WaitExited will wait for the specified container to state exit, subject // to a maximum time limit in seconds supplied by the caller func WaitExited(t testing.TB, name string, timeout time.Duration, cmdOperators ...CmdOperator) { + t.Helper() waitForInspectResult(t, name, "{{.State.Status}}", "exited", timeout, cmdOperators...) } From 959889efd920e8ee3b50098f73671ffccc71de0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 13 Jul 2023 15:06:36 +0200 Subject: [PATCH 139/293] integration: Don't env cleanup before parallel subtests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calling function returned from setupTest (which calls testEnv.Clean) in a defer block inside a test that spawns parallel subtests caused the cleanup function to be called before any of the subtest did anything. Change the defer expressions to use `t.Cleanup` instead to call it only after all subtests have also finished. This only changes tests which have parallel subtests. Signed-off-by: Paweł Gronowski (cherry picked from commit f9e2eed55d014bd29dc54a7300a10be72222fa5f) Signed-off-by: Paweł Gronowski --- integration/build/build_test.go | 4 ++-- integration/container/container_test.go | 2 +- integration/container/create_test.go | 8 ++++---- integration/container/devices_windows_test.go | 2 +- integration/container/mounts_linux_test.go | 2 +- integration/container/stop_linux_test.go | 2 +- integration/container/stop_windows_test.go | 2 +- integration/container/wait_test.go | 8 ++++---- integration/network/network_test.go | 4 ++-- integration/plugin/common/plugin_test.go | 2 +- integration/volume/volume_test.go | 2 +- 11 files changed, 19 insertions(+), 19 deletions(-) diff --git a/integration/build/build_test.go b/integration/build/build_test.go index 589ebc0a7706e..3dad23108ebbc 100644 --- a/integration/build/build_test.go +++ b/integration/build/build_test.go @@ -22,7 +22,7 @@ import ( ) func TestBuildWithRemoveAndForceRemove(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) cases := []struct { name string @@ -577,7 +577,7 @@ COPY --from=intermediate C:\\stuff C:\\stuff func TestBuildWithEmptyDockerfile(t *testing.T) { skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "broken in earlier versions") ctx := context.TODO() - defer setupTest(t)() + t.Cleanup(setupTest(t)) tests := []struct { name string diff --git a/integration/container/container_test.go b/integration/container/container_test.go index 81c5bb7685a3d..e822667615fb4 100644 --- a/integration/container/container_test.go +++ b/integration/container/container_test.go @@ -13,7 +13,7 @@ import ( // TestContainerInvalidJSON tests that POST endpoints that expect a body return // the correct error when sending invalid JSON requests. func TestContainerInvalidJSON(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) // POST endpoints that accept / expect a JSON body; endpoints := []string{ diff --git a/integration/container/create_test.go b/integration/container/create_test.go index eabb2a69b51d1..d52bb1f98889b 100644 --- a/integration/container/create_test.go +++ b/integration/container/create_test.go @@ -25,7 +25,7 @@ import ( ) func TestCreateFailsWhenIdentifierDoesNotExist(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) client := testEnv.APIClient() testCases := []struct { @@ -91,7 +91,7 @@ func TestCreateLinkToNonExistingContainer(t *testing.T) { } func TestCreateWithInvalidEnv(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) client := testEnv.APIClient() testCases := []struct { @@ -337,7 +337,7 @@ func TestCreateWithCustomReadonlyPaths(t *testing.T) { } func TestCreateWithInvalidHealthcheckParams(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) client := testEnv.APIClient() ctx := context.Background() @@ -532,7 +532,7 @@ func TestCreatePlatformSpecificImageNoPlatform(t *testing.T) { func TestCreateInvalidHostConfig(t *testing.T) { skip.If(t, testEnv.DaemonInfo.OSType == "windows") - defer setupTest(t)() + t.Cleanup(setupTest(t)) apiClient := testEnv.APIClient() ctx := context.Background() diff --git a/integration/container/devices_windows_test.go b/integration/container/devices_windows_test.go index 1ab5c6e690fcb..a966694194fee 100644 --- a/integration/container/devices_windows_test.go +++ b/integration/container/devices_windows_test.go @@ -18,7 +18,7 @@ import ( // via HostConfig.Devices through to the implementation in hcsshim. func TestWindowsDevices(t *testing.T) { skip.If(t, testEnv.DaemonInfo.OSType != "windows") - defer setupTest(t)() + t.Cleanup(setupTest(t)) client := testEnv.APIClient() ctx := context.Background() diff --git a/integration/container/mounts_linux_test.go b/integration/container/mounts_linux_test.go index 405e16c1dbea3..6df3e8a3b8529 100644 --- a/integration/container/mounts_linux_test.go +++ b/integration/container/mounts_linux_test.go @@ -90,7 +90,7 @@ func TestContainerNetworkMountsNoChown(t *testing.T) { func TestMountDaemonRoot(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon) - defer setupTest(t)() + t.Cleanup(setupTest(t)) client := testEnv.APIClient() ctx := context.Background() info, err := client.Info(ctx) diff --git a/integration/container/stop_linux_test.go b/integration/container/stop_linux_test.go index 9cfe42351d7e7..a26fdbcd3ba8e 100644 --- a/integration/container/stop_linux_test.go +++ b/integration/container/stop_linux_test.go @@ -27,7 +27,7 @@ import ( // a timeout works as documented, i.e. in case of negative timeout // waiting is not limited (issue #35311). func TestStopContainerWithTimeout(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) client := testEnv.APIClient() ctx := context.Background() diff --git a/integration/container/stop_windows_test.go b/integration/container/stop_windows_test.go index 65683822e97aa..27bcdce54c74b 100644 --- a/integration/container/stop_windows_test.go +++ b/integration/container/stop_windows_test.go @@ -18,7 +18,7 @@ import ( // waiting is not limited (issue #35311). func TestStopContainerWithTimeout(t *testing.T) { skip.If(t, testEnv.OSType == "windows") - defer setupTest(t)() + t.Cleanup(setupTest(t)) client := testEnv.APIClient() ctx := context.Background() diff --git a/integration/container/wait_test.go b/integration/container/wait_test.go index 9140faa2f61a0..47b5640575e38 100644 --- a/integration/container/wait_test.go +++ b/integration/container/wait_test.go @@ -16,7 +16,7 @@ import ( ) func TestWaitNonBlocked(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) cli := request.NewAPIClient(t) testCases := []struct { @@ -59,7 +59,7 @@ func TestWaitBlocked(t *testing.T) { // Windows busybox does not support trap in this way, not sleep with sub-second // granularity. It will always exit 0x40010004. skip.If(t, testEnv.DaemonInfo.OSType != "linux") - defer setupTest(t)() + t.Cleanup(setupTest(t)) cli := request.NewAPIClient(t) testCases := []struct { @@ -104,7 +104,7 @@ func TestWaitBlocked(t *testing.T) { } func TestWaitConditions(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) cli := request.NewAPIClient(t) testCases := []struct { @@ -179,7 +179,7 @@ func TestWaitConditions(t *testing.T) { } func TestWaitRestartedContainer(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) cli := request.NewAPIClient(t) testCases := []struct { diff --git a/integration/network/network_test.go b/integration/network/network_test.go index bfc6e2998e315..2c005028d6ae3 100644 --- a/integration/network/network_test.go +++ b/integration/network/network_test.go @@ -67,7 +67,7 @@ func TestRunContainerWithBridgeNone(t *testing.T) { // TestNetworkInvalidJSON tests that POST endpoints that expect a body return // the correct error when sending invalid JSON requests. func TestNetworkInvalidJSON(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) // POST endpoints that accept / expect a JSON body; endpoints := []string{ @@ -126,7 +126,7 @@ func TestNetworkInvalidJSON(t *testing.T) { // TestNetworkList verifies that /networks returns a list of networks either // with, or without a trailing slash (/networks/). Regression test for https://github.com/moby/moby/issues/24595 func TestNetworkList(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) endpoints := []string{ "/networks", diff --git a/integration/plugin/common/plugin_test.go b/integration/plugin/common/plugin_test.go index 630d4a09c3455..75edcf33ce5b1 100644 --- a/integration/plugin/common/plugin_test.go +++ b/integration/plugin/common/plugin_test.go @@ -32,7 +32,7 @@ import ( // TestPluginInvalidJSON tests that POST endpoints that expect a body return // the correct error when sending invalid JSON requests. func TestPluginInvalidJSON(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) // POST endpoints that accept / expect a JSON body; endpoints := []string{ diff --git a/integration/volume/volume_test.go b/integration/volume/volume_test.go index 96bc1e1117d28..2d6d2f31c43a1 100644 --- a/integration/volume/volume_test.go +++ b/integration/volume/volume_test.go @@ -196,7 +196,7 @@ func TestVolumesInspect(t *testing.T) { // TestVolumesInvalidJSON tests that POST endpoints that expect a body return // the correct error when sending invalid JSON requests. func TestVolumesInvalidJSON(t *testing.T) { - defer setupTest(t)() + t.Cleanup(setupTest(t)) // POST endpoints that accept / expect a JSON body; endpoints := []string{"/volumes/create"} From fee4db80a0b03b6abf4f37a9849dd4221dfd28d8 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 14 Jul 2023 18:56:47 +0200 Subject: [PATCH 140/293] client: TestSetHostHeader: don't use un-keyed literals Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 2a59188760b3062436c482bb40fd153df2c0dc44) Signed-off-by: Sebastiaan van Stijn --- client/request_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/client/request_test.go b/client/request_test.go index 6e5a6e81f21c1..a9506d312d8c8 100644 --- a/client/request_test.go +++ b/client/request_test.go @@ -28,24 +28,24 @@ func TestSetHostHeader(t *testing.T) { expectedURLHost string }{ { - "unix:///var/run/docker.sock", - "docker", - "/var/run/docker.sock", + host: "unix:///var/run/docker.sock", + expectedHost: "docker", + expectedURLHost: "/var/run/docker.sock", }, { - "npipe:////./pipe/docker_engine", - "docker", - "//./pipe/docker_engine", + host: "npipe:////./pipe/docker_engine", + expectedHost: "docker", + expectedURLHost: "//./pipe/docker_engine", }, { - "tcp://0.0.0.0:4243", - "", - "0.0.0.0:4243", + host: "tcp://0.0.0.0:4243", + expectedHost: "", + expectedURLHost: "0.0.0.0:4243", }, { - "tcp://localhost:4243", - "", - "localhost:4243", + host: "tcp://localhost:4243", + expectedHost: "", + expectedURLHost: "localhost:4243", }, } From 597a5f9794a5c46f405397154727b5193973dd13 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 12 Jul 2023 14:15:38 +0200 Subject: [PATCH 141/293] client: define a "dummy" hostname to use for local connections For local communications (npipe://, unix://), the hostname is not used, but we need valid and meaningful hostname. The current code used the client's `addr` as hostname in some cases, which could contain the path for the unix-socket (`/var/run/docker.sock`), which gets rejected by go1.20.6 and go1.19.11 because of a security fix for [CVE-2023-29406 ][1], which was implemented in https://go.dev/issue/60374. Prior versions go Go would clean the host header, and strip slashes in the process, but go1.20.6 and go1.19.11 no longer do, and reject the host header. This patch introduces a `DummyHost` const, and uses this dummy host for cases where we don't need an actual hostname. Before this patch (using go1.20.6): make GO_VERSION=1.20.6 TEST_FILTER=TestAttach test-integration === RUN TestAttachWithTTY attach_test.go:46: assertion failed: error is not nil: http: invalid Host header --- FAIL: TestAttachWithTTY (0.11s) === RUN TestAttachWithoutTTy attach_test.go:46: assertion failed: error is not nil: http: invalid Host header --- FAIL: TestAttachWithoutTTy (0.02s) FAIL With this patch applied: make GO_VERSION=1.20.6 TEST_FILTER=TestAttach test-integration INFO: Testing against a local daemon === RUN TestAttachWithTTY --- PASS: TestAttachWithTTY (0.12s) === RUN TestAttachWithoutTTy --- PASS: TestAttachWithoutTTy (0.02s) PASS [1]: https://github.com/advisories/GHSA-f8f7-69v5-w4vx Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 92975f0c11f0566cc3c36659f5e3bb9faf5cb176) Signed-off-by: Sebastiaan van Stijn --- client/client.go | 30 ++++++++++++++++++++++++++++++ client/hijack.go | 6 +++++- client/request.go | 10 ++++------ client/request_test.go | 4 ++-- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/client/client.go b/client/client.go index 1c081a51ae692..54fa36cca88e7 100644 --- a/client/client.go +++ b/client/client.go @@ -56,6 +56,36 @@ import ( "github.com/pkg/errors" ) +// DummyHost is a hostname used for local communication. +// +// It acts as a valid formatted hostname for local connections (such as "unix://" +// or "npipe://") which do not require a hostname. It should never be resolved, +// but uses the special-purpose ".localhost" TLD (as defined in [RFC 2606, Section 2] +// and [RFC 6761, Section 6.3]). +// +// [RFC 7230, Section 5.4] defines that an empty header must be used for such +// cases: +// +// If the authority component is missing or undefined for the target URI, +// then a client MUST send a Host header field with an empty field-value. +// +// However, [Go stdlib] enforces the semantics of HTTP(S) over TCP, does not +// allow an empty header to be used, and requires req.URL.Scheme to be either +// "http" or "https". +// +// For further details, refer to: +// +// - https://github.com/docker/engine-api/issues/189 +// - https://github.com/golang/go/issues/13624 +// - https://github.com/golang/go/issues/61076 +// - https://github.com/moby/moby/issues/45935 +// +// [RFC 2606, Section 2]: https://www.rfc-editor.org/rfc/rfc2606.html#section-2 +// [RFC 6761, Section 6.3]: https://www.rfc-editor.org/rfc/rfc6761#section-6.3 +// [RFC 7230, Section 5.4]: https://datatracker.ietf.org/doc/html/rfc7230#section-5.4 +// [Go stdlib]: https://github.com/golang/go/blob/6244b1946bc2101b01955468f1be502dbadd6807/src/net/http/transport.go#L558-L569 +const DummyHost = "api.moby.localhost" + // ErrRedirect is the error returned by checkRedirect when the request is non-GET. var ErrRedirect = errors.New("unexpected redirect in response") diff --git a/client/hijack.go b/client/hijack.go index 6bdacab10adbe..4dcaaca4c58f5 100644 --- a/client/hijack.go +++ b/client/hijack.go @@ -64,7 +64,11 @@ func fallbackDial(proto, addr string, tlsConfig *tls.Config) (net.Conn, error) { } func (cli *Client) setupHijackConn(ctx context.Context, req *http.Request, proto string) (net.Conn, string, error) { - req.Host = cli.addr + req.URL.Host = cli.addr + if cli.proto == "unix" || cli.proto == "npipe" { + // Override host header for non-tcp connections. + req.Host = DummyHost + } req.Header.Set("Connection", "Upgrade") req.Header.Set("Upgrade", proto) diff --git a/client/request.go b/client/request.go index c799095c12272..bcedcf3bd9d44 100644 --- a/client/request.go +++ b/client/request.go @@ -96,16 +96,14 @@ func (cli *Client) buildRequest(method, path string, body io.Reader, headers hea return nil, err } req = cli.addHeaders(req, headers) + req.URL.Scheme = cli.scheme + req.URL.Host = cli.addr if cli.proto == "unix" || cli.proto == "npipe" { - // For local communications, it doesn't matter what the host is. We just - // need a valid and meaningful host name. (See #189) - req.Host = "docker" + // Override host header for non-tcp connections. + req.Host = DummyHost } - req.URL.Host = cli.addr - req.URL.Scheme = cli.scheme - if expectedPayload && req.Header.Get("Content-Type") == "" { req.Header.Set("Content-Type", "text/plain") } diff --git a/client/request_test.go b/client/request_test.go index a9506d312d8c8..feaf9bf0338e7 100644 --- a/client/request_test.go +++ b/client/request_test.go @@ -29,12 +29,12 @@ func TestSetHostHeader(t *testing.T) { }{ { host: "unix:///var/run/docker.sock", - expectedHost: "docker", + expectedHost: DummyHost, expectedURLHost: "/var/run/docker.sock", }, { host: "npipe:////./pipe/docker_engine", - expectedHost: "docker", + expectedHost: DummyHost, expectedURLHost: "//./pipe/docker_engine", }, { From 547ea18fbbea3d14514120ed67c6e8aff18eb347 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 12 Jul 2023 15:07:59 +0200 Subject: [PATCH 142/293] pkg/plugins: use a dummy hostname for local connections For local communications (npipe://, unix://), the hostname is not used, but we need valid and meaningful hostname. The current code used the socket path as hostname, which gets rejected by go1.20.6 and go1.19.11 because of a security fix for [CVE-2023-29406 ][1], which was implemented in https://go.dev/issue/60374. Prior versions go Go would clean the host header, and strip slashes in the process, but go1.20.6 and go1.19.11 no longer do, and reject the host header. Before this patch, tests would fail on go1.20.6: === FAIL: pkg/authorization TestAuthZRequestPlugin (15.01s) time="2023-07-12T12:53:45Z" level=warning msg="Unable to connect to plugin: //tmp/authz2422457390/authz-test-plugin.sock/AuthZPlugin.AuthZReq: Post \"http://%2F%2Ftmp%2Fauthz2422457390%2Fauthz-test-plugin.sock/AuthZPlugin.AuthZReq\": http: invalid Host header, retrying in 1s" time="2023-07-12T12:53:46Z" level=warning msg="Unable to connect to plugin: //tmp/authz2422457390/authz-test-plugin.sock/AuthZPlugin.AuthZReq: Post \"http://%2F%2Ftmp%2Fauthz2422457390%2Fauthz-test-plugin.sock/AuthZPlugin.AuthZReq\": http: invalid Host header, retrying in 2s" time="2023-07-12T12:53:48Z" level=warning msg="Unable to connect to plugin: //tmp/authz2422457390/authz-test-plugin.sock/AuthZPlugin.AuthZReq: Post \"http://%2F%2Ftmp%2Fauthz2422457390%2Fauthz-test-plugin.sock/AuthZPlugin.AuthZReq\": http: invalid Host header, retrying in 4s" time="2023-07-12T12:53:52Z" level=warning msg="Unable to connect to plugin: //tmp/authz2422457390/authz-test-plugin.sock/AuthZPlugin.AuthZReq: Post \"http://%2F%2Ftmp%2Fauthz2422457390%2Fauthz-test-plugin.sock/AuthZPlugin.AuthZReq\": http: invalid Host header, retrying in 8s" authz_unix_test.go:82: Failed to authorize request Post "http://%2F%2Ftmp%2Fauthz2422457390%2Fauthz-test-plugin.sock/AuthZPlugin.AuthZReq": http: invalid Host header [1]: https://github.com/advisories/GHSA-f8f7-69v5-w4vx Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 6b7705d5b29e226a24902a8dcc488836faaee33c) Signed-off-by: Sebastiaan van Stijn --- pkg/plugins/client.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/plugins/client.go b/pkg/plugins/client.go index 752fecd0ae471..e683eb777d727 100644 --- a/pkg/plugins/client.go +++ b/pkg/plugins/client.go @@ -18,6 +18,12 @@ import ( const ( defaultTimeOut = 30 + + // dummyHost is a hostname used for local communication. + // + // For local communications (npipe://, unix://), the hostname is not used, + // but we need valid and meaningful hostname. + dummyHost = "plugin.moby.localhost" ) func newTransport(addr string, tlsConfig *tlsconfig.Options) (transport.Transport, error) { @@ -44,8 +50,12 @@ func newTransport(addr string, tlsConfig *tlsconfig.Options) (transport.Transpor return nil, err } scheme := httpScheme(u) - - return transport.NewHTTPTransport(tr, scheme, socket), nil + hostName := u.Host + if hostName == "" || u.Scheme == "unix" || u.Scheme == "npipe" { + // Override host header for non-tcp connections. + hostName = dummyHost + } + return transport.NewHTTPTransport(tr, scheme, hostName), nil } // NewClient creates a new plugin client (http). From bdaadec7881c56835a31473cbe6cd165b816e4b0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 12 Jul 2023 17:37:01 +0200 Subject: [PATCH 143/293] testutil: use dummyhost for non-tcp connections Signed-off-by: Sebastiaan van Stijn (cherry picked from commit e1db9e9848435042922512b3b5f6dc97f627af00) Signed-off-by: Sebastiaan van Stijn --- integration-cli/docker_api_attach_test.go | 5 +++++ testutil/request/request.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/integration-cli/docker_api_attach_test.go b/integration-cli/docker_api_attach_test.go index 6d31c51ec344a..401087ab2eaea 100644 --- a/integration-cli/docker_api_attach_test.go +++ b/integration-cli/docker_api_attach_test.go @@ -236,6 +236,11 @@ func requestHijack(method, endpoint string, data io.Reader, ct, daemon string, m req.URL.Scheme = "http" req.URL.Host = hostURL.Host + if hostURL.Scheme == "unix" || hostURL.Scheme == "npipe" { + // Override host header for non-tcp connections. + req.Host = client.DummyHost + } + for _, opt := range modifiers { opt(req) } diff --git a/testutil/request/request.go b/testutil/request/request.go index d5f559c666370..6a91dc9b37514 100644 --- a/testutil/request/request.go +++ b/testutil/request/request.go @@ -125,6 +125,11 @@ func newRequest(endpoint string, opts *Options) (*http.Request, error) { } req.URL.Host = hostURL.Host + if hostURL.Scheme == "unix" || hostURL.Scheme == "npipe" { + // Override host header for non-tcp connections. + req.Host = client.DummyHost + } + for _, config := range opts.requestModifiers { if err := config(req); err != nil { return nil, err From fa909dfaf4e181037b5e3d6c8288d864cfd57133 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 12 Jul 2023 14:30:01 +0200 Subject: [PATCH 144/293] update go to go1.20.6 go1.20.6 (released 2023-07-11) includes a security fix to the net/http package, as well as bug fixes to the compiler, cgo, the cover tool, the go command, the runtime, and the crypto/ecdsa, go/build, go/printer, net/mail, and text/template packages. See the Go 1.20.6 milestone on our issue tracker for details. https://github.com/golang/go/issues?q=milestone%3AGo1.20.6+label%3ACherryPickApproved Full diff: https://github.com/golang/go/compare/go1.20.5...go1.20.6 These minor releases include 1 security fixes following the security policy: net/http: insufficient sanitization of Host header The HTTP/1 client did not fully validate the contents of the Host header. A maliciously crafted Host header could inject additional headers or entire requests. The HTTP/1 client now refuses to send requests containing an invalid Request.Host or Request.URL.Host value. Thanks to Bartek Nowotarski for reporting this issue. Includes security fixes for [CVE-2023-29406 ][1] and Go issue https://go.dev/issue/60374 [1]: https://github.com/advisories/GHSA-f8f7-69v5-w4vx Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 1ead2dd35d36a9ec77f5560eb048bd6243ea8757) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/.windows.yml | 2 +- .github/workflows/test.yml | 2 +- Dockerfile | 2 +- Dockerfile.simple | 2 +- Dockerfile.windows | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml index a5465dcc200c9..66812324a817d 100644 --- a/.github/workflows/.windows.yml +++ b/.github/workflows/.windows.yml @@ -15,7 +15,7 @@ on: default: false env: - GO_VERSION: "1.20.5" + GO_VERSION: "1.20.6" GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 WINDOWS_BASE_IMAGE: mcr.microsoft.com/windows/servercore diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6556709203bba..6d5fb2f6a8c5c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ on: pull_request: env: - GO_VERSION: "1.20.5" + GO_VERSION: "1.20.6" GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 ITG_CLI_MATRIX_SIZE: 6 diff --git a/Dockerfile b/Dockerfile index 3a80a248fa1ca..98098bff2721c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20.5 +ARG GO_VERSION=1.20.6 ARG BASE_DEBIAN_DISTRO="bullseye" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" ARG XX_VERSION=1.2.1 diff --git a/Dockerfile.simple b/Dockerfile.simple index 0431db9b80b20..dd6fabfe16867 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -5,7 +5,7 @@ # This represents the bare minimum required to build and test Docker. -ARG GO_VERSION=1.20.5 +ARG GO_VERSION=1.20.6 ARG BASE_DEBIAN_DISTRO="bullseye" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" diff --git a/Dockerfile.windows b/Dockerfile.windows index 0383d11ba9919..68b4b74830a49 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -165,7 +165,7 @@ FROM microsoft/windowsservercore # Use PowerShell as the default shell SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -ARG GO_VERSION=1.20.5 +ARG GO_VERSION=1.20.6 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 ARG CONTAINERD_VERSION=v1.7.1 From 75a90f85ad7e0ab94ab82a0c9c788110e9bbe698 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 14 Jul 2023 22:39:33 +0200 Subject: [PATCH 145/293] gha: add note about buildkit using older go version Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 41f235a2f8b9d67b044f65607a0d6de56142f783) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/buildkit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index f0f64a576fcd3..1a27364a7193b 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -13,6 +13,7 @@ on: pull_request: env: + # FIXME(thaJeztah): update to newer go versions once BuildKit's vendoring has the fix from https://github.com/moby/moby/pull/45942 GO_VERSION: "1.20.5" DESTDIR: ./build From 632fc235d65dcc2ddf7d887c86f09b3ccb55656c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:09:41 +0200 Subject: [PATCH 146/293] builder/dockerfile: use string-literals for easier grep'ing Use string-literal for reduce escaped quotes, which makes for easier grepping. While at it, also changed http -> https to keep some linters at bay. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 202907b14ca1a41341d2b128ce2f1a1cd8b2aa5e) Signed-off-by: Sebastiaan van Stijn --- builder/dockerfile/copy_test.go | 24 ++++++++++++------------ builder/dockerfile/copy_windows.go | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/builder/dockerfile/copy_test.go b/builder/dockerfile/copy_test.go index 5fce0bd4eb32b..e837ce07adce4 100644 --- a/builder/dockerfile/copy_test.go +++ b/builder/dockerfile/copy_test.go @@ -53,31 +53,31 @@ func TestGetFilenameForDownload(t *testing.T) { expected string }{ { - path: "http://www.example.com/", + path: "https://www.example.com/", expected: "", }, { - path: "http://www.example.com/xyz", + path: "https://www.example.com/xyz", expected: "xyz", }, { - path: "http://www.example.com/xyz.html", + path: "https://www.example.com/xyz.html", expected: "xyz.html", }, { - path: "http://www.example.com/xyz/", + path: "https://www.example.com/xyz/", expected: "", }, { - path: "http://www.example.com/xyz/uvw", + path: "https://www.example.com/xyz/uvw", expected: "uvw", }, { - path: "http://www.example.com/xyz/uvw.html", + path: "https://www.example.com/xyz/uvw.html", expected: "uvw.html", }, { - path: "http://www.example.com/xyz/uvw/", + path: "https://www.example.com/xyz/uvw/", expected: "", }, { @@ -114,23 +114,23 @@ func TestGetFilenameForDownload(t *testing.T) { expected: "xyz.html", }, { - disposition: "attachment; filename=\"xyz\"", + disposition: `attachment; filename="xyz"`, expected: "xyz", }, { - disposition: "attachment; filename=\"xyz.html\"", + disposition: `attachment; filename="xyz.html"`, expected: "xyz.html", }, { - disposition: "attachment; filename=\"/xyz.html\"", + disposition: `attachment; filename="/xyz.html"`, expected: "xyz.html", }, { - disposition: "attachment; filename=\"/xyz/uvw\"", + disposition: `attachment; filename="/xyz/uvw"`, expected: "uvw", }, { - disposition: "attachment; filename=\"Naïve file.txt\"", + disposition: `attachment; filename="Naïve file.txt"`, expected: "Naïve file.txt", }, } diff --git a/builder/dockerfile/copy_windows.go b/builder/dockerfile/copy_windows.go index 1a3a488516970..bca088da6e20e 100644 --- a/builder/dockerfile/copy_windows.go +++ b/builder/dockerfile/copy_windows.go @@ -15,8 +15,8 @@ import ( ) var pathDenyList = map[string]bool{ - "c:\\": true, - "c:\\windows": true, + `c:\`: true, + `c:\windows`: true, } func init() { From 5bba60b1bb4115e8cf9485b9747caa1d1555ab58 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:11:23 +0200 Subject: [PATCH 147/293] builder/builder-next: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 2f6162033938f4f5d470d1a14c15f9ee740b6168) Signed-off-by: Sebastiaan van Stijn --- builder/builder-next/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index d21d88eeacbb0..46fc83eb7402b 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -304,7 +304,7 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt return nil, errors.Errorf("snapshotter doesn't support differ") } - leases, err := lm.List(ctx, "labels.\"buildkit/lease.temporary\"") + leases, err := lm.List(ctx, `labels."buildkit/lease.temporary"`) if err != nil { return nil, err } From a3f1f4eeb09d125dcf0371011631e9179a971b0b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:12:16 +0200 Subject: [PATCH 148/293] integration-cli: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 6331a3a3465b4c3d97bfefc3faa6ea68d59f391c) Signed-off-by: Sebastiaan van Stijn --- integration-cli/check_test.go | 2 +- integration-cli/docker_api_containers_test.go | 2 +- integration-cli/docker_cli_build_test.go | 4 +-- integration-cli/docker_cli_commit_test.go | 22 +++++++------- integration-cli/docker_cli_events_test.go | 4 +-- .../docker_cli_events_unix_test.go | 30 +++++++++---------- integration-cli/docker_cli_links_test.go | 2 +- integration-cli/docker_cli_ps_test.go | 6 ++-- integration-cli/docker_cli_pull_local_test.go | 4 +-- integration-cli/docker_cli_run_test.go | 4 +-- .../docker_cli_service_logs_test.go | 4 +-- 11 files changed, 42 insertions(+), 42 deletions(-) diff --git a/integration-cli/check_test.go b/integration-cli/check_test.go index dd036f3f0e01b..ebfc5603ae093 100644 --- a/integration-cli/check_test.go +++ b/integration-cli/check_test.go @@ -76,7 +76,7 @@ func printCliVersion() { cli.SetTestEnvironment(testEnv) cmd := cli.Docker(cli.Args("version")) if cmd.Error != nil { - fmt.Printf("WARNING: Failed to run \"docker version\": %+v\n", cmd.Error) + fmt.Printf("WARNING: Failed to run 'docker version': %+v\n", cmd.Error) return } diff --git a/integration-cli/docker_api_containers_test.go b/integration-cli/docker_api_containers_test.go index d89c605f5111b..a5ebda5916174 100644 --- a/integration-cli/docker_api_containers_test.go +++ b/integration-cli/docker_api_containers_test.go @@ -1158,7 +1158,7 @@ func (s *DockerAPISuite) TestContainerAPIDeleteRemoveLinks(c *testing.T) { assert.Assert(c, waitRun(id2) == nil) links := inspectFieldJSON(c, id2, "HostConfig.Links") - assert.Equal(c, links, "[\"/tlink1:/tlink2/tlink1\"]", "expected to have links between containers") + assert.Equal(c, links, `["/tlink1:/tlink2/tlink1"]`, "expected to have links between containers") removeOptions := types.ContainerRemoveOptions{ RemoveLinks: true, diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index ce5ed94c02364..3f37290eba0c2 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -2258,7 +2258,7 @@ docker.com>" `)) res := inspectField(c, name, "Author") - if res != "\"Docker IO \"" { + if res != `"Docker IO "` { c.Fatalf("Parsed string did not match the escaped string. Got: %q", res) } } @@ -2272,7 +2272,7 @@ func (s *DockerCLIBuildSuite) TestBuildVerifyIntString(c *testing.T) { MAINTAINER 123`)) out, _ := dockerCmd(c, "inspect", name) - if !strings.Contains(out, "\"123\"") { + if !strings.Contains(out, `"123"`) { c.Fatalf("Output does not contain the int as a string:\n%s", out) } } diff --git a/integration-cli/docker_cli_commit_test.go b/integration-cli/docker_cli_commit_test.go index c562430527e7a..817eb99e1ea57 100644 --- a/integration-cli/docker_cli_commit_test.go +++ b/integration-cli/docker_cli_commit_test.go @@ -120,17 +120,17 @@ func (s *DockerCLICommitSuite) TestCommitChange(c *testing.T) { dockerCmd(c, "run", "--name", "test", "busybox", "true") imageID, _ := dockerCmd(c, "commit", - "--change", "EXPOSE 8080", - "--change", "ENV DEBUG true", - "--change", "ENV test 1", - "--change", "ENV PATH /foo", - "--change", "LABEL foo bar", - "--change", "CMD [\"/bin/sh\"]", - "--change", "WORKDIR /opt", - "--change", "ENTRYPOINT [\"/bin/sh\"]", - "--change", "USER testuser", - "--change", "VOLUME /var/lib/docker", - "--change", "ONBUILD /usr/local/bin/python-build --dir /app/src", + "--change", `EXPOSE 8080`, + "--change", `ENV DEBUG true`, + "--change", `ENV test 1`, + "--change", `ENV PATH /foo`, + "--change", `LABEL foo bar`, + "--change", `CMD ["/bin/sh"]`, + "--change", `WORKDIR /opt`, + "--change", `ENTRYPOINT ["/bin/sh"]`, + "--change", `USER testuser`, + "--change", `VOLUME /var/lib/docker`, + "--change", `ONBUILD /usr/local/bin/python-build --dir /app/src`, "test", "test-commit") imageID = strings.TrimSpace(imageID) diff --git a/integration-cli/docker_cli_events_test.go b/integration-cli/docker_cli_events_test.go index f517ffa5a01c7..9884a202af314 100644 --- a/integration-cli/docker_cli_events_test.go +++ b/integration-cli/docker_cli_events_test.go @@ -750,7 +750,7 @@ func (s *DockerCLIEventSuite) TestEventsFormatBadFunc(c *testing.T) { result.Assert(c, icmd.Expected{ Error: "exit status 64", ExitCode: 64, - Err: "Error parsing format: template: :1: function \"badFuncString\" not defined", + Err: `Error parsing format: template: :1: function "badFuncString" not defined`, }) } @@ -760,6 +760,6 @@ func (s *DockerCLIEventSuite) TestEventsFormatBadField(c *testing.T) { result.Assert(c, icmd.Expected{ Error: "exit status 64", ExitCode: 64, - Err: "Error parsing format: template: :1:2: executing \"\" at <.badFieldString>: can't evaluate field badFieldString in type *events.Message", + Err: `Error parsing format: template: :1:2: executing "" at <.badFieldString>: can't evaluate field badFieldString in type *events.Message`, }) } diff --git a/integration-cli/docker_cli_events_unix_test.go b/integration-cli/docker_cli_events_unix_test.go index 355dda72c293b..69f14744b4132 100644 --- a/integration-cli/docker_cli_events_unix_test.go +++ b/integration-cli/docker_cli_events_unix_test.go @@ -415,21 +415,21 @@ func (s *DockerDaemonSuite) TestDaemonEvents(c *testing.T) { // only check for values known (daemon ID/name) or explicitly set above, // otherwise just check for names being present. expectedSubstrings := []string{ - " daemon reload " + info.ID + " ", - "(allow-nondistributable-artifacts=[", - " debug=true, ", - " default-ipc-mode=", - " default-runtime=", - " default-shm-size=", - " insecure-registries=[", - " labels=[\"bar=foo\"], ", - " live-restore=", - " max-concurrent-downloads=1, ", - " max-concurrent-uploads=5, ", - " name=" + info.Name, - " registry-mirrors=[", - " runtimes=", - " shutdown-timeout=10)", + ` daemon reload ` + info.ID + " ", + `(allow-nondistributable-artifacts=[`, + ` debug=true, `, + ` default-ipc-mode=`, + ` default-runtime=`, + ` default-shm-size=`, + ` insecure-registries=[`, + ` labels=["bar=foo"], `, + ` live-restore=`, + ` max-concurrent-downloads=1, `, + ` max-concurrent-uploads=5, `, + ` name=` + info.Name, + ` registry-mirrors=[`, + ` runtimes=`, + ` shutdown-timeout=10)`, } for _, s := range expectedSubstrings { diff --git a/integration-cli/docker_cli_links_test.go b/integration-cli/docker_cli_links_test.go index bdcd4c59a5798..1b253b6876ebf 100644 --- a/integration-cli/docker_cli_links_test.go +++ b/integration-cli/docker_cli_links_test.go @@ -218,7 +218,7 @@ func (s *DockerCLILinksSuite) TestLinkShortDefinition(c *testing.T) { assert.Assert(c, waitRun(cid2) == nil) links := inspectFieldJSON(c, cid2, "HostConfig.Links") - assert.Equal(c, links, "[\"/shortlinkdef:/link2/shortlinkdef\"]") + assert.Equal(c, links, `["/shortlinkdef:/link2/shortlinkdef"]`) } func (s *DockerCLILinksSuite) TestLinksNetworkHostContainer(c *testing.T) { diff --git a/integration-cli/docker_cli_ps_test.go b/integration-cli/docker_cli_ps_test.go index 212ddb626193c..4742ad4be3123 100644 --- a/integration-cli/docker_cli_ps_test.go +++ b/integration-cli/docker_cli_ps_test.go @@ -176,7 +176,7 @@ func (s *DockerCLIPsSuite) TestPsListContainersSize(c *testing.T) { select { case <-wait: case <-time.After(3 * time.Second): - c.Fatalf("Calling \"docker ps -s\" timed out!") + c.Fatalf(`Calling "docker ps -s" timed out!`) } result.Assert(c, icmd.Success) lines := strings.Split(strings.Trim(result.Combined(), "\n "), "\n") @@ -634,8 +634,8 @@ func (s *DockerCLIPsSuite) TestPsShowMounts(c *testing.T) { var bindMountSource string var bindMountDestination string if DaemonIsWindows() { - bindMountSource = "c:\\" - bindMountDestination = "c:\\t" + bindMountSource = `c:\` + bindMountDestination = `c:\t` } else { bindMountSource = "/tmp" bindMountDestination = "/t" diff --git a/integration-cli/docker_cli_pull_local_test.go b/integration-cli/docker_cli_pull_local_test.go index aa7852a275e0b..af48b55c801d5 100644 --- a/integration-cli/docker_cli_pull_local_test.go +++ b/integration-cli/docker_cli_pull_local_test.go @@ -391,7 +391,7 @@ func (s *DockerRegistryAuthHtpasswdSuite) TestPullWithExternalAuthLoginWithSchem b, err := os.ReadFile(configPath) assert.NilError(c, err) - assert.Assert(c, !strings.Contains(string(b), "\"auth\":")) + assert.Assert(c, !strings.Contains(string(b), `"auth":`)) dockerCmd(c, "--config", tmp, "tag", "busybox", repoName) dockerCmd(c, "--config", tmp, "push", repoName) @@ -433,7 +433,7 @@ func (s *DockerRegistryAuthHtpasswdSuite) TestPullWithExternalAuth(c *testing.T) b, err := os.ReadFile(configPath) assert.NilError(c, err) - assert.Assert(c, !strings.Contains(string(b), "\"auth\":")) + assert.Assert(c, !strings.Contains(string(b), `"auth":`)) dockerCmd(c, "--config", tmp, "tag", "busybox", repoName) dockerCmd(c, "--config", tmp, "push", repoName) diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index 2469106be6adb..171189204367e 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -1673,7 +1673,7 @@ func (s *DockerCLIRunSuite) TestRunCopyVolumeUIDGID(c *testing.T) { RUN mkdir -p /hello && touch /hello/test && chown dockerio.dockerio /hello`)) // Test that the uid and gid is copied from the image to the volume - out, _ := dockerCmd(c, "run", "--rm", "-v", "/hello", name, "sh", "-c", "ls -l / | grep hello | awk '{print $3\":\"$4}'") + out, _ := dockerCmd(c, "run", "--rm", "-v", "/hello", name, "sh", "-c", `ls -l / | grep hello | awk '{print $3":"$4}'`) out = strings.TrimSpace(out) if out != "dockerio:dockerio" { c.Fatalf("Wrong /hello ownership: %s, expected dockerio:dockerio", out) @@ -2053,7 +2053,7 @@ func (s *DockerCLIRunSuite) TestRunPortInUse(c *testing.T) { c.Fatalf("Binding on used port must fail") } if !strings.Contains(out, "port is already allocated") { - c.Fatalf("Out must be about \"port is already allocated\", got %s", out) + c.Fatalf(`Out must be about "port is already allocated", got %s`, out) } } diff --git a/integration-cli/docker_cli_service_logs_test.go b/integration-cli/docker_cli_service_logs_test.go index 34f472a42aff7..100425d34a086 100644 --- a/integration-cli/docker_cli_service_logs_test.go +++ b/integration-cli/docker_cli_service_logs_test.go @@ -230,7 +230,7 @@ func (s *DockerSwarmSuite) TestServiceLogsTaskLogs(c *testing.T) { assert.Assert(c, id != "") // so, right here, we're basically inspecting by id and returning only // the ID. if they don't match, the service doesn't exist. - result = icmd.RunCmd(d.Command("service", "inspect", "--format=\"{{.ID}}\"", id)) + result = icmd.RunCmd(d.Command("service", "inspect", `--format="{{.ID}}"`, id)) result.Assert(c, icmd.Expected{Out: id}) // make sure task has been deployed. @@ -283,7 +283,7 @@ func (s *DockerSwarmSuite) TestServiceLogsTTY(c *testing.T) { assert.Assert(c, id != "") // so, right here, we're basically inspecting by id and returning only // the ID. if they don't match, the service doesn't exist. - result = icmd.RunCmd(d.Command("service", "inspect", "--format=\"{{.ID}}\"", id)) + result = icmd.RunCmd(d.Command("service", "inspect", `--format="{{.ID}}"`, id)) result.Assert(c, icmd.Expected{Out: id}) // make sure task has been deployed. From 147b87a03ea17fdf36b09a2bed6be0d95b916e83 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:13:09 +0200 Subject: [PATCH 149/293] daemon: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 02815416bb1b084ddc0e3049fcc90739a41b0921) Signed-off-by: Sebastiaan van Stijn --- daemon/daemon_unix.go | 6 +++--- daemon/daemon_unix_test.go | 2 +- daemon/network.go | 2 +- daemon/top_unix.go | 2 +- daemon/top_unix_test.go | 4 ++-- daemon/update.go | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 065dee9e0d49c..b0df9229a2b68 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -701,7 +701,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. if hostConfig.CgroupParent != "" && UsingSystemd(daemon.configStore) { // CgroupParent for systemd cgroup should be named as "xxx.slice" if len(hostConfig.CgroupParent) <= 6 || !strings.HasSuffix(hostConfig.CgroupParent, ".slice") { - return warnings, fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"") + return warnings, fmt.Errorf(`cgroup-parent for systemd cgroup should be a valid slice named as "xxx.slice"`) } } if hostConfig.Runtime == "" { @@ -754,7 +754,7 @@ func verifyDaemonSettings(conf *config.Config) error { } if conf.CgroupParent != "" && UsingSystemd(conf) { if len(conf.CgroupParent) <= 6 || !strings.HasSuffix(conf.CgroupParent, ".slice") { - return fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"") + return fmt.Errorf(`cgroup-parent for systemd cgroup should be a valid slice named as "xxx.slice"`) } } @@ -1069,7 +1069,7 @@ func initBridgeDriver(controller *libnetwork.Controller, config *config.Config) libnetwork.NetworkOptionIpam("default", "", v4Conf, v6Conf, nil), libnetwork.NetworkOptionDeferIPv6Alloc(deferIPv6Alloc)) if err != nil { - return fmt.Errorf("Error creating default \"bridge\" network: %v", err) + return fmt.Errorf(`error creating default "bridge" network: %v`, err) } return nil } diff --git a/daemon/daemon_unix_test.go b/daemon/daemon_unix_test.go index f44c0be48f547..03d04a579c2d3 100644 --- a/daemon/daemon_unix_test.go +++ b/daemon/daemon_unix_test.go @@ -148,7 +148,7 @@ func TestParseSecurityOptWithDeprecatedColon(t *testing.T) { t.Fatalf("Unexpected parseSecurityOpt error: %v", err) } if opts.AppArmorProfile != "test_profile" { - t.Fatalf("Unexpected AppArmorProfile, expected: \"test_profile\", got %q", opts.AppArmorProfile) + t.Fatalf(`Unexpected AppArmorProfile, expected: "test_profile", got %q`, opts.AppArmorProfile) } // test seccomp diff --git a/daemon/network.go b/daemon/network.go index 3bf8852d4fc25..ade5bbc80b54c 100644 --- a/daemon/network.go +++ b/daemon/network.go @@ -375,7 +375,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string if err != nil { if errors.Is(err, libnetwork.ErrDataStoreNotInitialized) { //nolint: revive - return nil, errors.New("This node is not a swarm manager. Use \"docker swarm init\" or \"docker swarm join\" to connect this node to swarm and try again.") + return nil, errors.New(`This node is not a swarm manager. Use "docker swarm init" or "docker swarm join" to connect this node to swarm and try again.`) } return nil, err } diff --git a/daemon/top_unix.go b/daemon/top_unix.go index 68da2596e45ee..e2cc38c3a484d 100644 --- a/daemon/top_unix.go +++ b/daemon/top_unix.go @@ -29,7 +29,7 @@ func validatePSArgs(psArgs string) error { k := group[1] v := group[2] if k != "pid" { - return fmt.Errorf("specifying \"%s=%s\" is not allowed", k, v) + return fmt.Errorf(`specifying "%s=%s" is not allowed`, k, v) } } } diff --git a/daemon/top_unix_test.go b/daemon/top_unix_test.go index a663323b67549..4bf4d23408550 100644 --- a/daemon/top_unix_test.go +++ b/daemon/top_unix_test.go @@ -12,8 +12,8 @@ import ( func TestContainerTopValidatePSArgs(t *testing.T) { tests := map[string]bool{ "ae -o uid=PID": true, - "ae -o \"uid= PID\"": true, // ascii space (0x20) - "ae -o \"uid= PID\"": false, // unicode space (U+2003, 0xe2 0x80 0x83) + `ae -o "uid= PID"`: true, // ascii space (0x20) + `ae -o "uid= PID"`: false, // unicode space (U+2003, 0xe2 0x80 0x83) "ae o uid=PID": true, "aeo uid=PID": true, "ae -O uid=PID": true, diff --git a/daemon/update.go b/daemon/update.go index f01635e49e859..fb88bc6f807c2 100644 --- a/daemon/update.go +++ b/daemon/update.go @@ -53,7 +53,7 @@ func (daemon *Daemon) update(name string, hostConfig *container.HostConfig) erro if ctr.RemovalInProgress || ctr.Dead { ctr.Unlock() - return errCannotUpdate(ctr.ID, fmt.Errorf("container is marked for removal and cannot be \"update\"")) + return errCannotUpdate(ctr.ID, fmt.Errorf(`container is marked for removal and cannot be "update"`)) } if err := ctr.UpdateContainer(hostConfig); err != nil { From 892857179ab6d1d3f2f1833af9b82c5b6fe3e789 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:13:53 +0200 Subject: [PATCH 150/293] cli/debug: use string-literals for easier grep'ing Also removed some newlines from t.Fatal() as they shouldn't be needed. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit c3d533f37fae8b81adbce5627214562fd26646c4) Signed-off-by: Sebastiaan van Stijn --- cli/debug/debug_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/debug/debug_test.go b/cli/debug/debug_test.go index 5b6d788a39847..72028a8e36dc3 100644 --- a/cli/debug/debug_test.go +++ b/cli/debug/debug_test.go @@ -14,20 +14,20 @@ func TestEnable(t *testing.T) { }() Enable() if os.Getenv("DEBUG") != "1" { - t.Fatalf("expected DEBUG=1, got %s\n", os.Getenv("DEBUG")) + t.Fatalf("expected DEBUG=1, got %s", os.Getenv("DEBUG")) } if logrus.GetLevel() != logrus.DebugLevel { - t.Fatalf("expected log level %v, got %v\n", logrus.DebugLevel, logrus.GetLevel()) + t.Fatalf("expected log level %v, got %v", logrus.DebugLevel, logrus.GetLevel()) } } func TestDisable(t *testing.T) { Disable() if os.Getenv("DEBUG") != "" { - t.Fatalf("expected DEBUG=\"\", got %s\n", os.Getenv("DEBUG")) + t.Fatalf(`expected DEBUG="", got %s`, os.Getenv("DEBUG")) } if logrus.GetLevel() != logrus.InfoLevel { - t.Fatalf("expected log level %v, got %v\n", logrus.InfoLevel, logrus.GetLevel()) + t.Fatalf("expected log level %v, got %v", logrus.InfoLevel, logrus.GetLevel()) } } From ae8e3294dd32662c51b77d8f6380793f6fda0ec3 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:14:27 +0200 Subject: [PATCH 151/293] client: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 4e69e16fde6c7f7949143139fb492467d9b9a7ec) Signed-off-by: Sebastiaan van Stijn --- client/image_tag_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/image_tag_test.go b/client/image_tag_test.go index 63653af20744e..a0c14f813a656 100644 --- a/client/image_tag_test.go +++ b/client/image_tag_test.go @@ -42,7 +42,7 @@ func TestImageTagInvalidSourceImageName(t *testing.T) { } err := client.ImageTag(context.Background(), "invalid_source_image_name_", "repo:tag") - if err == nil || err.Error() != "Error parsing reference: \"invalid_source_image_name_\" is not a valid repository/tag: invalid reference format" { + if err == nil || err.Error() != `Error parsing reference: "invalid_source_image_name_" is not a valid repository/tag: invalid reference format` { t.Fatalf("expected Parsing Reference Error, got %v", err) } } From 2d2df4376b004619f589f387532b5d9f9e714430 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:22:14 +0200 Subject: [PATCH 152/293] daemon/cluster: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 0db4a32b9c8ad2ad36bf0f3603dc23fddba22269) Signed-off-by: Sebastiaan van Stijn --- daemon/cluster/cluster.go | 2 +- daemon/cluster/errors.go | 6 +++--- daemon/cluster/services.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/daemon/cluster/cluster.go b/daemon/cluster/cluster.go index c39001c51c710..86c144e773e1e 100644 --- a/daemon/cluster/cluster.go +++ b/daemon/cluster/cluster.go @@ -359,7 +359,7 @@ func (c *Cluster) errNoManager(st nodeState) error { if st.err == errSwarmCertificatesExpired { return errSwarmCertificatesExpired } - return errors.WithStack(notAvailableError("This node is not a swarm manager. Use \"docker swarm init\" or \"docker swarm join\" to connect this node to swarm and try again.")) + return errors.WithStack(notAvailableError(`This node is not a swarm manager. Use "docker swarm init" or "docker swarm join" to connect this node to swarm and try again.`)) } if st.swarmNode.Manager() != nil { return errors.WithStack(notAvailableError("This node is not a swarm manager. Manager is being prepared or has trouble connecting to the cluster.")) diff --git a/daemon/cluster/errors.go b/daemon/cluster/errors.go index 9ec716b1bac65..1371b9ab521a6 100644 --- a/daemon/cluster/errors.go +++ b/daemon/cluster/errors.go @@ -5,13 +5,13 @@ const ( errNoSwarm notAvailableError = "This node is not part of a swarm" // errSwarmExists is returned on initialize or join request for a cluster that has already been activated - errSwarmExists notAvailableError = "This node is already part of a swarm. Use \"docker swarm leave\" to leave this swarm and join another one." + errSwarmExists notAvailableError = `This node is already part of a swarm. Use "docker swarm leave" to leave this swarm and join another one.` // errSwarmJoinTimeoutReached is returned when cluster join could not complete before timeout was reached. - errSwarmJoinTimeoutReached notAvailableError = "Timeout was reached before node joined. The attempt to join the swarm will continue in the background. Use the \"docker info\" command to see the current swarm status of your node." + errSwarmJoinTimeoutReached notAvailableError = `Timeout was reached before node joined. The attempt to join the swarm will continue in the background. Use the "docker info" command to see the current swarm status of your node.` // errSwarmLocked is returned if the swarm is encrypted and needs a key to unlock it. - errSwarmLocked notAvailableError = "Swarm is encrypted and needs to be unlocked before it can be used. Please use \"docker swarm unlock\" to unlock it." + errSwarmLocked notAvailableError = `Swarm is encrypted and needs to be unlocked before it can be used. Please use "docker swarm unlock" to unlock it.` // errSwarmCertificatesExpired is returned if docker was not started for the whole validity period and they had no chance to renew automatically. errSwarmCertificatesExpired notAvailableError = "Swarm certificates have expired. To replace them, leave the swarm and join again." diff --git a/daemon/cluster/services.go b/daemon/cluster/services.go index 12af0cbf4f165..201eba156cdb3 100644 --- a/daemon/cluster/services.go +++ b/daemon/cluster/services.go @@ -456,7 +456,7 @@ func (c *Cluster) ServiceLogs(ctx context.Context, selector *backend.LogSelector } else { t, err := strconv.Atoi(config.Tail) if err != nil { - return nil, errors.New("tail value must be a positive integer or \"all\"") + return nil, errors.New(`tail value must be a positive integer or "all"`) } if t < 0 { return nil, errors.New("negative tail values not supported") From efe9e90ef5239b257f3a359dea0e52c44bab1b04 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:23:03 +0200 Subject: [PATCH 153/293] libnetwork: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 96a1c444ccceb4bb6a2e41dbd5185277b72588ad) Signed-off-by: Sebastiaan van Stijn --- libnetwork/store_linux_test.go | 2 +- libnetwork/store_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libnetwork/store_linux_test.go b/libnetwork/store_linux_test.go index f6213540fb29f..bbb9e5e45c9bc 100644 --- a/libnetwork/store_linux_test.go +++ b/libnetwork/store_linux_test.go @@ -24,7 +24,7 @@ func TestNoPersist(t *testing.T) { defer ctrl.Stop() nw, err := ctrl.NewNetwork("host", "host", "", NetworkOptionPersist(false)) if err != nil { - t.Fatalf("Error creating default \"host\" network: %v", err) + t.Fatalf(`Error creating default "host" network: %v`, err) } ep, err := nw.CreateEndpoint("newendpoint", []EndpointOption{}...) if err != nil { diff --git a/libnetwork/store_test.go b/libnetwork/store_test.go index c857aeb63fc84..adfe3dff47170 100644 --- a/libnetwork/store_test.go +++ b/libnetwork/store_test.go @@ -30,7 +30,7 @@ func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Con defer ctrl.Stop() nw, err := ctrl.NewNetwork("host", "host", "") if err != nil { - t.Fatalf("Error creating default \"host\" network: %v", err) + t.Fatalf(`Error creating default "host" network: %v`, err) } ep, err := nw.CreateEndpoint("newendpoint", []EndpointOption{}...) if err != nil { From ff667ed932009f25c252c8dead8651d26353a40b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:23:57 +0200 Subject: [PATCH 154/293] integration: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ddec605aeffdb8b7abfe276c4a43437288a0fd80) Signed-off-by: Sebastiaan van Stijn --- integration/config/config_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/integration/config/config_test.go b/integration/config/config_test.go index a8c39ef295e61..9fcb8af2ff4af 100644 --- a/integration/config/config_test.go +++ b/integration/config/config_test.go @@ -251,9 +251,10 @@ func TestTemplatedConfig(t *testing.T) { Templating: &swarmtypes.Driver{ Name: "golang", }, - Data: []byte("SERVICE_NAME={{.Service.Name}}\n" + - "{{secret \"referencedsecrettarget\"}}\n" + - "{{config \"referencedconfigtarget\"}}\n"), + Data: []byte(`SERVICE_NAME={{.Service.Name}} +{{secret "referencedsecrettarget"}} +{{config "referencedconfigtarget"}} +`), } templatedConfig, err := c.ConfigCreate(ctx, configSpec) From 69d77bc150eeee3f466187e3c850375f80454f57 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:24:11 +0200 Subject: [PATCH 155/293] opts: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 84000190d30f715b27b6150bc9fd1a648830b3bc) Signed-off-by: Sebastiaan van Stijn --- opts/opts_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opts/opts_test.go b/opts/opts_test.go index fe4b7f5ce5c43..49ceea8676732 100644 --- a/opts/opts_test.go +++ b/opts/opts_test.go @@ -71,10 +71,10 @@ func TestListOptsWithoutValidator(t *testing.T) { t.Errorf("%d != 3", o.Len()) } if !o.Get("bar") { - t.Error("o.Get(\"bar\") == false") + t.Error(`o.Get("bar") == false`) } if o.Get("baz") { - t.Error("o.Get(\"baz\") == true") + t.Error(`o.Get("baz") == true`) } o.Delete("foo") if o.String() != "[bar bar]" { @@ -106,10 +106,10 @@ func TestListOptsWithValidator(t *testing.T) { t.Errorf("%d != 1", o.Len()) } if !o.Get("max-file=2") { - t.Error("o.Get(\"max-file=2\") == false") + t.Error(`o.Get("max-file=2") == false`) } if o.Get("baz") { - t.Error("o.Get(\"baz\") == true") + t.Error(`o.Get("baz") == true`) } o.Delete("max-file=2") if o.String() != "" { @@ -361,7 +361,7 @@ func TestMapMapOpts(t *testing.T) { o.Set("r1=k12=v12") assert.Check(t, is.DeepEqual(tmpMap["r1"], map[string]string{"k11": "v11", "k12": "v12"})) - if o.Set("invalid-key={\"k\":\"v\"}") == nil { + if o.Set(`invalid-key={"k":"v"}`) == nil { t.Error("validator is not being called") } } From cea5829402f2b3ebf1e35c4b04e4a0b2bc2b78d2 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:24:24 +0200 Subject: [PATCH 156/293] pkg/idtools: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 1da079f211396ea6030a60ca37848d3c1f7dd99b) Signed-off-by: Sebastiaan van Stijn --- pkg/idtools/idtools_unix_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/idtools/idtools_unix_test.go b/pkg/idtools/idtools_unix_test.go index 5a4acc25d11ad..d8f4661c14754 100644 --- a/pkg/idtools/idtools_unix_test.go +++ b/pkg/idtools/idtools_unix_test.go @@ -459,14 +459,14 @@ func TestLookupUserAndGroup(t *testing.T) { func TestLookupUserAndGroupThatDoesNotExist(t *testing.T) { fakeUser := "fakeuser" _, err := LookupUser(fakeUser) - assert.Check(t, is.Error(err, "getent unable to find entry \""+fakeUser+"\" in passwd database")) + assert.Check(t, is.Error(err, `getent unable to find entry "fakeuser" in passwd database`)) _, err = LookupUID(-1) assert.Check(t, is.ErrorContains(err, "")) fakeGroup := "fakegroup" _, err = LookupGroup(fakeGroup) - assert.Check(t, is.Error(err, "getent unable to find entry \""+fakeGroup+"\" in group database")) + assert.Check(t, is.Error(err, `getent unable to find entry "fakegroup" in group database`)) _, err = LookupGID(-1) assert.Check(t, is.ErrorContains(err, "")) From 962a4f434f482e1709bc5f4cac54e4931a12647d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:24:39 +0200 Subject: [PATCH 157/293] pkg/ioutils: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit fded42c3bdd43a8e4f02140626dd0a403a6fece5) Signed-off-by: Sebastiaan van Stijn --- pkg/ioutils/buffer_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/ioutils/buffer_test.go b/pkg/ioutils/buffer_test.go index b8887bfde0496..4df20cb5b4827 100644 --- a/pkg/ioutils/buffer_test.go +++ b/pkg/ioutils/buffer_test.go @@ -66,7 +66,7 @@ func TestFixedBufferString(t *testing.T) { out := buf.String() if out != "helloworld" { - t.Fatalf("expected output to be \"helloworld\", got %q", out) + t.Fatalf(`expected output to be "helloworld", got %q`, out) } // read 5 bytes @@ -76,7 +76,7 @@ func TestFixedBufferString(t *testing.T) { // test that fixedBuffer.String() only returns the part that hasn't been read out = buf.String() if out != "world" { - t.Fatalf("expected output to be \"world\", got %q", out) + t.Fatalf(`expected output to be "world", got %q`, out) } } @@ -92,7 +92,7 @@ func TestFixedBufferWrite(t *testing.T) { } if string(buf.buf[:5]) != "hello" { - t.Fatalf("expected \"hello\", got %q", string(buf.buf[:5])) + t.Fatalf(`expected "hello", got %q`, string(buf.buf[:5])) } n, err = buf.Write(bytes.Repeat([]byte{1}, 64)) @@ -121,7 +121,7 @@ func TestFixedBufferRead(t *testing.T) { } if string(b) != "hello" { - t.Fatalf("expected \"hello\", got %q", string(b)) + t.Fatalf(`expected "hello", got %q`, string(b)) } n, err = buf.Read(b) @@ -134,7 +134,7 @@ func TestFixedBufferRead(t *testing.T) { } if string(b) != " worl" { - t.Fatalf("expected \" worl\", got %s", string(b)) + t.Fatalf(`expected " worl", got %s`, string(b)) } b = b[:1] @@ -148,6 +148,6 @@ func TestFixedBufferRead(t *testing.T) { } if string(b) != "d" { - t.Fatalf("expected \"d\", got %s", string(b)) + t.Fatalf(`expected "d", got %s`, string(b)) } } From deea880581e738b948d2dcd7d50fe890062ff2e2 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 5 Jul 2023 12:24:58 +0200 Subject: [PATCH 158/293] pkg/jsonmessage: use string-literals for easier grep'ing Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ec11aea880a8aaaab65e79bde49cb3d1280be073) Signed-off-by: Sebastiaan van Stijn --- pkg/jsonmessage/jsonmessage_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/jsonmessage/jsonmessage_test.go b/pkg/jsonmessage/jsonmessage_test.go index 45ec0a525fcfd..dd33d02b21925 100644 --- a/pkg/jsonmessage/jsonmessage_test.go +++ b/pkg/jsonmessage/jsonmessage_test.go @@ -244,22 +244,22 @@ func TestDisplayJSONMessagesStream(t *testing.T) { "", ""}, // Without progress & ID - "{ \"status\": \"status\" }": { + `{ "status": "status" }`: { "status\n", "status\n", }, // Without progress, with ID - "{ \"id\": \"ID\",\"status\": \"status\" }": { + `{ "id": "ID","status": "status" }`: { "ID: status\n", "ID: status\n", }, // With progress - "{ \"id\": \"ID\", \"status\": \"status\", \"progress\": \"ProgressMessage\" }": { + `{ "id": "ID", "status": "status", "progress": "ProgressMessage" }`: { "ID: status ProgressMessage", fmt.Sprintf("\n%c[%dAID: status ProgressMessage%c[%dB", 27, 1, 27, 1), }, // With progressDetail - "{ \"id\": \"ID\", \"status\": \"status\", \"progressDetail\": { \"Current\": 1} }": { + `{ "id": "ID", "status": "status", "progressDetail": { "Current": 1} }`: { "", // progressbar is disabled in non-terminal fmt.Sprintf("\n%c[%dA%c[2K\rID: status 1B\r%c[%dB", 27, 1, 27, 27, 1), }, From a5c0fda157c93d85280af88995722ef3f83ce8bf Mon Sep 17 00:00:00 2001 From: Albin Kerouanton Date: Sat, 8 Jul 2023 15:19:04 +0200 Subject: [PATCH 159/293] libnet: Return proper error when overlay network can't be created The commit befff0e13f68d39fed40e9f81bafc25042e944da inadvertendly disabled the error returned when trying to create an overlay network on a node which is not part of a Swarm cluster. Since commit e3708a89ccf5c88273660b2409e2ea1197530b0b the overlay netdriver returns the error: `no VNI provided`. This commit reinstate the original error message by checking if the node is a manager before calling libnetwork's `controller.NewNetwork()`. Signed-off-by: Albin Kerouanton (cherry picked from commit 21dcbada2d1383ccf771410048cb1574dfe43c6d) Signed-off-by: Sebastiaan van Stijn --- daemon/network.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/daemon/network.go b/daemon/network.go index ade5bbc80b54c..d89d87e1c4d38 100644 --- a/daemon/network.go +++ b/daemon/network.go @@ -291,6 +291,16 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string return nil, PredefinedNetworkError(create.Name) } + c := daemon.netController + driver := create.Driver + if driver == "" { + driver = c.Config().DefaultDriver + } + + if driver == "overlay" && !daemon.cluster.IsManager() && !agent { + return nil, errors.New(`This node is not a swarm manager. Use "docker swarm init" or "docker swarm join" to connect this node to swarm and try again.`) + } + var warning string nw, err := daemon.GetNetworkByName(create.Name) if err != nil { @@ -309,12 +319,6 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string warning = fmt.Sprintf("Network with name %s (id : %s) already exists", nw.Name(), nw.ID()) } - c := daemon.netController - driver := create.Driver - if driver == "" { - driver = c.Config().DefaultDriver - } - networkOptions := make(map[string]string) for k, v := range create.Options { networkOptions[k] = v @@ -373,10 +377,6 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string n, err := c.NewNetwork(driver, create.Name, id, nwOptions...) if err != nil { - if errors.Is(err, libnetwork.ErrDataStoreNotInitialized) { - //nolint: revive - return nil, errors.New(`This node is not a swarm manager. Use "docker swarm init" or "docker swarm join" to connect this node to swarm and try again.`) - } return nil, err } From 738d8417e049f2d55987cd759a88f48810e8487e Mon Sep 17 00:00:00 2001 From: Albin Kerouanton Date: Sat, 8 Jul 2023 15:31:27 +0200 Subject: [PATCH 160/293] libnet: Return a 403 when overlay network isn't allowed With this change, the API will now return a 403 instead of a 500 when trying to create an overlay network on a non-manager node. Signed-off-by: Albin Kerouanton (cherry picked from commit d29240d9eb289af20581db425ed3e20f057d2164) Signed-off-by: Sebastiaan van Stijn --- api/swagger.yaml | 4 +++- daemon/network.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index a616794c1e2f3..a820f996f94f6 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -9896,7 +9896,9 @@ paths: Id: "22be93d5babb089c5aab8dbc369042fad48ff791584ca2da2100db837a1c7c30" Warning: "" 403: - description: "operation not supported for pre-defined networks" + description: | + Forbidden operation. This happens when trying to create a network named after a pre-defined network, + or when trying to create an overlay network on a daemon which is not part of a Swarm cluster. schema: $ref: "#/definitions/ErrorResponse" 404: diff --git a/daemon/network.go b/daemon/network.go index d89d87e1c4d38..e98a841cd32d3 100644 --- a/daemon/network.go +++ b/daemon/network.go @@ -298,7 +298,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string } if driver == "overlay" && !daemon.cluster.IsManager() && !agent { - return nil, errors.New(`This node is not a swarm manager. Use "docker swarm init" or "docker swarm join" to connect this node to swarm and try again.`) + return nil, errdefs.Forbidden(errors.New(`This node is not a swarm manager. Use "docker swarm init" or "docker swarm join" to connect this node to swarm and try again.`)) } var warning string From 68c0cec77257a4bdb6cc33ae6d1ccde7d98e5b1e Mon Sep 17 00:00:00 2001 From: Albin Kerouanton Date: Sun, 9 Jul 2023 22:26:07 +0200 Subject: [PATCH 161/293] Disable tls when launching dockerd through hack/make.sh The daemon sleeps for 15 seconds at start up when the API binds to a TCP socket with no TLS certificate set. That's what the hack/make/run script does, but it doesn't explicitly disable tls, thus we're experiencing this annoying delay every time we use this script. Signed-off-by: Albin Kerouanton (cherry picked from commit 6b1b71ced496bf35b5b9fedf8e0490ce91541ea3) Signed-off-by: Sebastiaan van Stijn --- hack/make/run | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/make/run b/hack/make/run index 87fe6d06aa724..16d9febc34275 100644 --- a/hack/make/run +++ b/hack/make/run @@ -58,6 +58,7 @@ args=( --host="unix://${socket}" --storage-driver="${DOCKER_GRAPHDRIVER}" --userland-proxy="${DOCKER_USERLANDPROXY}" + --tls=false $storage_params $extra_params ) From fcb87e8ae1dd78f28b7f2983b6eeb28cb0d3891b Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 9 Jul 2023 02:23:54 +0200 Subject: [PATCH 162/293] ci: push bin image to Docker Hub Signed-off-by: CrazyMax (cherry picked from commit 41261ea4ecd640555fc80fa340f0db5e0730c682) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/bin-image.yml | 113 +++++++++++++++++++++++++------- 1 file changed, 89 insertions(+), 24 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index 9a7e3790327ad..cc1530a9aaef6 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -15,6 +15,7 @@ on: pull_request: env: + MOBYBIN_REPO_SLUG: moby/moby-bin PLATFORM: Moby Engine PRODUCT: Moby DEFAULT_PRODUCT_LICENSE: Moby @@ -37,12 +38,29 @@ jobs: id: platforms run: | echo "matrix=$(docker buildx bake bin-image-cross --print | jq -cr '.target."bin-image-cross".platforms')" >>${GITHUB_OUTPUT} + + build: + runs-on: ubuntu-20.04 + needs: + - validate-dco + - prepare + strategy: + fail-fast: false + matrix: + platform: ${{ fromJson(needs.prepare.outputs.platforms) }} + steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Docker meta id: meta uses: docker/metadata-action@v4 with: - images: moby-bin + images: | + ${{ env.MOBYBIN_REPO_SLUG }} ### versioning strategy ## push semver tag v23.0.0 # moby/moby-bin:23.0.0 @@ -69,22 +87,59 @@ jobs: path: /tmp/bake-meta.json if-no-files-found: error retention-days: 1 + - + name: Remove tags from meta bake definition + run: | + # we just want labels being set in this job + jq -r 'del(.target."docker-metadata-action".tags)' "/tmp/bake-meta.json" > "${{ steps.meta.outputs.bake-file }}" + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_MOBYBIN_USERNAME }} + password: ${{ secrets.DOCKERHUB_MOBYBIN_TOKEN }} + - + name: Build + id: bake + uses: docker/bake-action@v3 + with: + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: bin-image + set: | + *.platform=${{ matrix.platform }} + *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} + - + name: Export digest + if: github.event_name != 'pull_request' + run: | + mkdir -p /tmp/digests + digest="${{ fromJSON(steps.bake.outputs.metadata).image['containerimage.digest'] }}" + touch "/tmp/digests/${digest#sha256:}" + - + name: Upload digest + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v3 + with: + name: digests + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 - build: + merge: runs-on: ubuntu-20.04 + if: github.event_name != 'pull_request' needs: - - validate-dco - - prepare - strategy: - fail-fast: false - matrix: - platform: ${{ fromJson(needs.prepare.outputs.platforms) }} + - build steps: - - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Download meta bake definition uses: actions/download-artifact@v3 @@ -92,19 +147,29 @@ jobs: name: bake-meta path: /tmp - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + name: Download digests + uses: actions/download-artifact@v3 + with: + name: digests + path: /tmp/digests - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Build - uses: docker/bake-action@v2 + name: Login to Docker Hub + uses: docker/login-action@v2 with: - files: | - ./docker-bake.hcl - /tmp/bake-meta.json - targets: bin-image - set: | - *.platform=${{ matrix.platform }} - *.output=type=cacheonly + username: ${{ secrets.DOCKERHUB_MOBYBIN_USERNAME }} + password: ${{ secrets.DOCKERHUB_MOBYBIN_TOKEN }} + - + name: Create manifest list and push + working-directory: /tmp/digests + run: | + set -x + docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map("-t " + .) | join(" ")' /tmp/bake.json) \ + $(printf '${{ env.MOBYBIN_REPO_SLUG }}@sha256:%s ' *) + - + name: Inspect image + run: | + set -x + docker buildx imagetools inspect ${{ env.MOBYBIN_REPO_SLUG }}:$(jq -cr '.target."docker-metadata-action".args.DOCKER_META_VERSION' /tmp/bake.json) From e7c333cb6eee9dc615a2bb90116b8e61dad4895c Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 11 Jul 2023 19:57:53 +0200 Subject: [PATCH 163/293] ci(bin-image): don't set tags when pushing by digest Signed-off-by: CrazyMax (cherry picked from commit 16865405940f3517edda213e36dea979ee649f81) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/bin-image.yml | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index cc1530a9aaef6..ac3891ede8972 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -75,23 +75,6 @@ jobs: type=semver,pattern={{version}} type=ref,event=branch type=ref,event=pr - - - name: Rename meta bake definition file - run: | - mv "${{ steps.meta.outputs.bake-file }}" "/tmp/bake-meta.json" - - - name: Upload meta bake definition - uses: actions/upload-artifact@v3 - with: - name: bake-meta - path: /tmp/bake-meta.json - if-no-files-found: error - retention-days: 1 - - - name: Remove tags from meta bake definition - run: | - # we just want labels being set in this job - jq -r 'del(.target."docker-metadata-action".tags)' "/tmp/bake-meta.json" > "${{ steps.meta.outputs.bake-file }}" - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -117,6 +100,7 @@ jobs: set: | *.platform=${{ matrix.platform }} *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} + *.tags= - name: Export digest if: github.event_name != 'pull_request' @@ -133,6 +117,18 @@ jobs: path: /tmp/digests/* if-no-files-found: error retention-days: 1 + - + name: Rename meta bake definition file + run: | + mv "${{ steps.meta.outputs.bake-file }}" "/tmp/bake-meta.json" + - + name: Upload meta bake definition + uses: actions/upload-artifact@v3 + with: + name: bake-meta + path: /tmp/bake-meta.json + if-no-files-found: error + retention-days: 1 merge: runs-on: ubuntu-20.04 From b9904ba3194f44566099dbd64dc03326cbc4fa37 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 11 Jul 2023 22:01:55 +0200 Subject: [PATCH 164/293] ci(bin-image): fix typo Signed-off-by: CrazyMax (cherry picked from commit 749d7449f9babe76c1cae7e276d72b67775f50f7) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/bin-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index ac3891ede8972..2b9161836d8a6 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -106,7 +106,7 @@ jobs: if: github.event_name != 'pull_request' run: | mkdir -p /tmp/digests - digest="${{ fromJSON(steps.bake.outputs.metadata).image['containerimage.digest'] }}" + digest="${{ fromJSON(steps.bake.outputs.metadata)['bin-image']['containerimage.digest'] }}" touch "/tmp/digests/${digest#sha256:}" - name: Upload digest From 26a457e7a31075c9fe4e89ce9f6b6803564646f4 Mon Sep 17 00:00:00 2001 From: Kevin Alvarez Date: Tue, 11 Jul 2023 22:43:49 +0200 Subject: [PATCH 165/293] ci(bin-image): fix meta step We can't upload the same file in a matrix so generate metadata in prepare job instead. Also fixes wrong bake meta file in merge job. Signed-off-by: CrazyMax (cherry picked from commit 0a126a85a466908a03ffaa52526a7ee5fe6b33d1) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/bin-image.yml | 74 ++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index 2b9161836d8a6..ab05844ab5e5f 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -33,6 +33,39 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.MOBYBIN_REPO_SLUG }} + ### versioning strategy + ## push semver tag v23.0.0 + # moby/moby-bin:23.0.0 + # moby/moby-bin:latest + ## push semver prelease tag v23.0.0-beta.1 + # moby/moby-bin:23.0.0-beta.1 + ## push on master + # moby/moby-bin:master + ## push on 23.0 branch + # moby/moby-bin:23.0 + tags: | + type=semver,pattern={{version}} + type=ref,event=branch + type=ref,event=pr + - + name: Rename meta bake definition file + run: | + mv "${{ steps.meta.outputs.bake-file }}" "/tmp/bake-meta.json" + - + name: Upload meta bake definition + uses: actions/upload-artifact@v3 + with: + name: bake-meta + path: /tmp/bake-meta.json + if-no-files-found: error + retention-days: 1 - name: Create platforms matrix id: platforms @@ -55,26 +88,11 @@ jobs: with: fetch-depth: 0 - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 + name: Download meta bake definition + uses: actions/download-artifact@v3 with: - images: | - ${{ env.MOBYBIN_REPO_SLUG }} - ### versioning strategy - ## push semver tag v23.0.0 - # moby/moby-bin:23.0.0 - # moby/moby-bin:latest - ## push semver prelease tag v23.0.0-beta.1 - # moby/moby-bin:23.0.0-beta.1 - ## push on master - # moby/moby-bin:master - ## push on 23.0 branch - # moby/moby-bin:23.0 - tags: | - type=semver,pattern={{version}} - type=ref,event=branch - type=ref,event=pr + name: bake-meta + path: /tmp - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -95,7 +113,7 @@ jobs: with: files: | ./docker-bake.hcl - ${{ steps.meta.outputs.bake-file }} + /tmp/bake-meta.json targets: bin-image set: | *.platform=${{ matrix.platform }} @@ -117,18 +135,6 @@ jobs: path: /tmp/digests/* if-no-files-found: error retention-days: 1 - - - name: Rename meta bake definition file - run: | - mv "${{ steps.meta.outputs.bake-file }}" "/tmp/bake-meta.json" - - - name: Upload meta bake definition - uses: actions/upload-artifact@v3 - with: - name: bake-meta - path: /tmp/bake-meta.json - if-no-files-found: error - retention-days: 1 merge: runs-on: ubuntu-20.04 @@ -162,10 +168,10 @@ jobs: working-directory: /tmp/digests run: | set -x - docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map("-t " + .) | join(" ")' /tmp/bake.json) \ + docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map("-t " + .) | join(" ")' /tmp/bake-meta.json) \ $(printf '${{ env.MOBYBIN_REPO_SLUG }}@sha256:%s ' *) - name: Inspect image run: | set -x - docker buildx imagetools inspect ${{ env.MOBYBIN_REPO_SLUG }}:$(jq -cr '.target."docker-metadata-action".args.DOCKER_META_VERSION' /tmp/bake.json) + docker buildx imagetools inspect ${{ env.MOBYBIN_REPO_SLUG }}:$(jq -cr '.target."docker-metadata-action".args.DOCKER_META_VERSION' /tmp/bake-meta.json) From 8bdf6d1baf7e1c0c7fab95d05ec7061aa03ffb2c Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Wed, 12 Jul 2023 07:04:41 -0600 Subject: [PATCH 166/293] ci(bin-image): add SHA-based tags Signed-off-by: Bjorn Neergaard (cherry picked from commit ecfa4f58666e22c27eb6e5cb5d01623b2d40df03) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/bin-image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index ab05844ab5e5f..182f02c04f999 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -50,10 +50,13 @@ jobs: # moby/moby-bin:master ## push on 23.0 branch # moby/moby-bin:23.0 + ## any push + # moby/moby-bin:sha-ad132f5 tags: | type=semver,pattern={{version}} type=ref,event=branch type=ref,event=pr + type=sha - name: Rename meta bake definition file run: | From bff68bf2cc6ea452dc2c246a6d859b21e7001de2 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 15 Jul 2023 02:37:03 +0200 Subject: [PATCH 167/293] client: Client.setupHijackConn: explicitly ignore errors Just making my IDE and some linters slightly happier. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit e11555218b65dfdf68d64e278fafb2967904d5e5) Signed-off-by: Sebastiaan van Stijn --- client/hijack.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/hijack.go b/client/hijack.go index 4dcaaca4c58f5..08f0744bd1493 100644 --- a/client/hijack.go +++ b/client/hijack.go @@ -84,8 +84,8 @@ func (cli *Client) setupHijackConn(ctx context.Context, req *http.Request, proto // state. Setting TCP KeepAlive on the socket connection will prohibit // ECONNTIMEOUT unless the socket connection truly is broken if tcpConn, ok := conn.(*net.TCPConn); ok { - tcpConn.SetKeepAlive(true) - tcpConn.SetKeepAlivePeriod(30 * time.Second) + _ = tcpConn.SetKeepAlive(true) + _ = tcpConn.SetKeepAlivePeriod(30 * time.Second) } clientconn := httputil.NewClientConn(conn, nil) @@ -100,7 +100,7 @@ func (cli *Client) setupHijackConn(ctx context.Context, req *http.Request, proto return nil, "", err } if resp.StatusCode != http.StatusSwitchingProtocols { - resp.Body.Close() + _ = resp.Body.Close() return nil, "", fmt.Errorf("unable to upgrade to %s, received %d", proto, resp.StatusCode) } } From d94f2dcab21f5d6ab82e8efb773acec3d9fec161 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 14 Jul 2023 19:50:41 +0200 Subject: [PATCH 168/293] client: Client.postHijacked: use Client.buildRequest Use Client.buildRequest instead of a local copy of the same logic so that we're using the same logic, and there's less chance of diverging. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit c219b09d4aeb71bc628bc8377b600db06fe71569) Signed-off-by: Sebastiaan van Stijn --- client/hijack.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/client/hijack.go b/client/hijack.go index 08f0744bd1493..7e84865f695f3 100644 --- a/client/hijack.go +++ b/client/hijack.go @@ -23,14 +23,10 @@ func (cli *Client) postHijacked(ctx context.Context, path string, query url.Valu if err != nil { return types.HijackedResponse{}, err } - - apiPath := cli.getAPIPath(ctx, path, query) - req, err := http.NewRequest(http.MethodPost, apiPath, bodyEncoded) + req, err := cli.buildRequest(http.MethodPost, cli.getAPIPath(ctx, path, query), bodyEncoded, headers) if err != nil { return types.HijackedResponse{}, err } - req = cli.addHeaders(req, headers) - conn, mediaType, err := cli.setupHijackConn(ctx, req, "tcp") if err != nil { return types.HijackedResponse{}, err @@ -64,11 +60,6 @@ func fallbackDial(proto, addr string, tlsConfig *tls.Config) (net.Conn, error) { } func (cli *Client) setupHijackConn(ctx context.Context, req *http.Request, proto string) (net.Conn, string, error) { - req.URL.Host = cli.addr - if cli.proto == "unix" || cli.proto == "npipe" { - // Override host header for non-tcp connections. - req.Host = DummyHost - } req.Header.Set("Connection", "Upgrade") req.Header.Set("Upgrade", proto) From 31567e0973faebb6d7a5a05d3df2ec6ac889c67b Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 3 Jan 2023 20:01:12 +0100 Subject: [PATCH 169/293] Dockerfile: use default apt mirrors Use default apt mirrors and also check APT_MIRROR is set before updating mirrors. Signed-off-by: CrazyMax (cherry picked from commit a1d2132bf6391b18cc225f0f113ff43efedece66) Signed-off-by: Bjorn Neergaard --- Dockerfile | 3 +-- Jenkinsfile | 11 +++++------ docker-bake.hcl | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98098bff2721c..1258caf999f53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,8 +33,7 @@ FROM --platform=$BUILDPLATFORM ${GOLANG_IMAGE} AS base COPY --from=xx / / RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache ARG APT_MIRROR -RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ - && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list +RUN test -n "$APT_MIRROR" && sed -ri "s/(httpredir|deb|security).debian.org/${APT_MIRROR}/g" /etc/apt/sources.list || true ARG DEBIAN_FRONTEND RUN apt-get update && apt-get install --no-install-recommends -y file ENV GO111MODULE=off diff --git a/Jenkinsfile b/Jenkinsfile index de00e5b31c965..6707839168792 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,6 @@ pipeline { DOCKER_BUILDKIT = '1' DOCKER_EXPERIMENTAL = '1' DOCKER_GRAPHDRIVER = 'overlay2' - APT_MIRROR = 'cdn-fastly.deb.debian.org' CHECK_CONFIG_COMMIT = '33a3680e08d1007e72c3b3f1454f823d8e9948ee' TESTDEBUG = '0' TIMEOUT = '120m' @@ -78,7 +77,7 @@ pipeline { stage("Build dev image") { steps { sh ''' - docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} . + docker build --force-rm -t docker:${GIT_COMMIT} . ''' } } @@ -191,7 +190,7 @@ pipeline { stage("Build dev image") { steps { sh ''' - docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} . + docker build --force-rm -t docker:${GIT_COMMIT} . ''' } } @@ -278,7 +277,7 @@ pipeline { stage("Build dev image") { steps { sh ''' - docker buildx build --load --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} . + docker buildx build --load --force-rm -t docker:${GIT_COMMIT} . ''' } } @@ -391,7 +390,7 @@ pipeline { stage("Build dev image") { steps { sh ''' - docker buildx build --load --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} . + docker buildx build --load --force-rm -t docker:${GIT_COMMIT} . ''' } } @@ -476,7 +475,7 @@ pipeline { } stage("Build dev image") { steps { - sh 'docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .' + sh 'docker build --force-rm -t docker:${GIT_COMMIT} .' } } stage("Unit tests") { diff --git a/docker-bake.hcl b/docker-bake.hcl index 0eb078aebcd6c..3d9675f184431 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,5 +1,5 @@ variable "APT_MIRROR" { - default = "cdn-fastly.deb.debian.org" + default = "" } variable "DOCKER_DEBUG" { default = "" From 151686a5c8cd3e863aefc74c7f29c8b60d5d498c Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 17 Jul 2023 09:52:05 -0600 Subject: [PATCH 170/293] Makefile: pass through APT_MIRROR This aligns `docker build` as invoked by the Makefile with both `docker buildx bake` as invoked by the Makefile and directly by the user. Signed-off-by: Bjorn Neergaard (cherry picked from commit bcea83ab9b705ba9655e34628586855f4749d793) Signed-off-by: Bjorn Neergaard --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 17aebbd26c6b7..aa1745369969e 100644 --- a/Makefile +++ b/Makefile @@ -140,6 +140,7 @@ endif DOCKER_RUN_DOCKER := $(DOCKER_FLAGS) "$(DOCKER_IMAGE)" DOCKER_BUILD_ARGS += --build-arg=GO_VERSION +DOCKER_BUILD_ARGS += --build-arg=APT_MIRROR DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_VERSION DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_REPOSITORY DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_INTEGRATION_VERSION From 05f82fdd0097da994c789729df8fd63a8777ea2f Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 17 Jul 2023 09:49:31 -0600 Subject: [PATCH 171/293] Dockerfile(.simple): align APT_MIRROR support Use a non-slash escape sequence to support mirrors with a path component, and do not unconditionally replace the mirror in Dockerfile.simple. Signed-off-by: Bjorn Neergaard (cherry picked from commit 235cd6c6b23874466179888b8d77ca573f76fe99) Signed-off-by: Bjorn Neergaard --- Dockerfile | 2 +- Dockerfile.simple | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1258caf999f53..7487f1409d574 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ FROM --platform=$BUILDPLATFORM ${GOLANG_IMAGE} AS base COPY --from=xx / / RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache ARG APT_MIRROR -RUN test -n "$APT_MIRROR" && sed -ri "s/(httpredir|deb|security).debian.org/${APT_MIRROR}/g" /etc/apt/sources.list || true +RUN test -n "$APT_MIRROR" && sed -ri "s#(httpredir|deb|security).debian.org#${APT_MIRROR}#g" /etc/apt/sources.list || true ARG DEBIAN_FRONTEND RUN apt-get update && apt-get install --no-install-recommends -y file ENV GO111MODULE=off diff --git a/Dockerfile.simple b/Dockerfile.simple index dd6fabfe16867..8605aa3fcd0c7 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -13,9 +13,9 @@ ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" FROM ${GOLANG_IMAGE} ENV GO111MODULE=off -# allow replacing httpredir or deb mirror -ARG APT_MIRROR=deb.debian.org -RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list +# allow replacing debian mirror +ARG APT_MIRROR +RUN test -n "$APT_MIRROR" && sed -ri "s#(httpredir|deb|security).debian.org#${APT_MIRROR}#g" /etc/apt/sources.list || true # Compile and runtime deps # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#build-dependencies From 0df2e1bdd89c62c4fe169fed69e59b86fa7fa26b Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 17 Jul 2023 09:50:27 -0600 Subject: [PATCH 172/293] Dockerfile: improve CLI/rootlesskit caching Use bind-mounts instead of a `COPY` for cli.sh, and use `COPY --link` for rootlesskit's build stage. Signed-off-by: Bjorn Neergaard (cherry picked from commit 12a19dcd84a2935b075524b29773aca412935de9) Signed-off-by: Bjorn Neergaard --- Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7487f1409d574..c8f56b6f7758a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -250,11 +250,11 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ FROM base AS dockercli WORKDIR /go/src/github.com/docker/cli -COPY hack/dockerfile/cli.sh /download-or-build-cli.sh ARG DOCKERCLI_REPOSITORY ARG DOCKERCLI_VERSION ARG TARGETPLATFORM -RUN --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,sharing=locked,target=./.git \ +RUN --mount=source=hack/dockerfile/cli.sh,target=/download-or-build-cli.sh \ + --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,sharing=locked,target=./.git \ --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM \ rm -f ./.git/*.lock \ && /download-or-build-cli.sh ${DOCKERCLI_VERSION} ${DOCKERCLI_REPOSITORY} /build \ @@ -262,12 +262,12 @@ RUN --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,sharing=locked,target=./ FROM base AS dockercli-integration WORKDIR /go/src/github.com/docker/cli -COPY hack/dockerfile/cli.sh /download-or-build-cli.sh ARG DOCKERCLI_INTEGRATION_REPOSITORY ARG DOCKERCLI_INTEGRATION_VERSION ARG TARGETPLATFORM -RUN --mount=type=cache,id=dockercli-integration-git-$TARGETPLATFORM,sharing=locked,target=./.git \ - --mount=type=cache,target=/root/.cache/go-build,id=dockercli-integration-build-$TARGETPLATFORM \ +RUN --mount=source=hack/dockerfile/cli.sh,target=/download-or-build-cli.sh \ + --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,sharing=locked,target=./.git \ + --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM \ rm -f ./.git/*.lock \ && /download-or-build-cli.sh ${DOCKERCLI_INTEGRATION_VERSION} ${DOCKERCLI_INTEGRATION_REPOSITORY} /build \ && /build/docker --version @@ -368,8 +368,8 @@ RUN --mount=from=rootlesskit-src,src=/usr/src/rootlesskit,rw \ xx-go build -o /build/rootlesskit-docker-proxy -ldflags="$([ "$DOCKER_STATIC" != "1" ] && echo "-linkmode=external")" ./cmd/rootlesskit-docker-proxy xx-verify $([ "$DOCKER_STATIC" = "1" ] && echo "--static") /build/rootlesskit-docker-proxy EOT -COPY ./contrib/dockerd-rootless.sh /build/ -COPY ./contrib/dockerd-rootless-setuptool.sh /build/ +COPY --link ./contrib/dockerd-rootless.sh /build/ +COPY --link ./contrib/dockerd-rootless-setuptool.sh /build/ FROM rootlesskit-build AS rootlesskit-linux FROM binary-dummy AS rootlesskit-windows From 544032f7a4df3c789353c7013e3db2903705bdec Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 17 Jul 2023 09:51:29 -0600 Subject: [PATCH 173/293] hack/d/cli.sh: properly handle errors in curl Add `-f` to output nothing to tar if the curl fails, and `-S` to report errors if they happen. Signed-off-by: Bjorn Neergaard (cherry picked from commit 780e8b233242690f7a1d5c393be24fdcd26e1a5b) Signed-off-by: Bjorn Neergaard --- hack/dockerfile/cli.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/dockerfile/cli.sh b/hack/dockerfile/cli.sh index f821994650270..9482736e552b0 100755 --- a/hack/dockerfile/cli.sh +++ b/hack/dockerfile/cli.sh @@ -9,7 +9,7 @@ DOWNLOAD_URL="https://download.docker.com/linux/static/stable/$(xx-info march)/d mkdir "$outdir" if curl --head --silent --fail "${DOWNLOAD_URL}" 1> /dev/null 2>&1; then - curl -Ls "${DOWNLOAD_URL}" | tar -xz docker/docker + curl -fsSL "${DOWNLOAD_URL}" | tar -xz docker/docker mv docker/docker "${outdir}/docker" else git init -q . From bd1ae65aab294c244fe29066a3172bfafa0de37f Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 17 Jul 2023 22:56:14 +0200 Subject: [PATCH 174/293] quota: remove gotest.tools from testhelpers gotest.tools has an init() which registers a '-update' flag; https://github.com/gotestyourself/gotest.tools/blob/a80f057529047c44e1a85d0d017b200787e537e0/internal/source/update.go#L21-L23 The quota helper contains a testhelpers file, which is meant for usage in (integration) tests, but as it's in the same pacakge as production code, would also trigger the gotest.tools init. This patch removes the gotest.tools code from this file. Before this patch: $ (exec -a libnetwork-setkey "$(which dockerd)" -help) Usage of libnetwork-setkey: -exec-root string docker exec root (default "/run/docker") -update update golden values With this patch applied: $ (exec -a libnetwork-setkey "$(which dockerd)" -help) Usage of libnetwork-setkey: -exec-root string docker exec root (default "/run/docker") Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 1aa17222e7a350971ae00ea75b34caaa9db382f6) Signed-off-by: Sebastiaan van Stijn --- quota/testhelpers.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/quota/testhelpers.go b/quota/testhelpers.go index 6087162e2ae78..72ec5ce510225 100644 --- a/quota/testhelpers.go +++ b/quota/testhelpers.go @@ -9,8 +9,6 @@ import ( "testing" "golang.org/x/sys/unix" - "gotest.tools/v3/assert" - "gotest.tools/v3/fs" ) const imageSize = 64 * 1024 * 1024 @@ -79,10 +77,7 @@ func WrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *test mountOptions = mountOptions + ",prjquota" } - mountPointDir := fs.NewDir(t, "xfs-mountPoint") - defer mountPointDir.Remove() - mountPoint := mountPointDir.Path() - + mountPoint := t.TempDir() out, err := exec.Command("mount", "-o", mountOptions, imageFileName, mountPoint).CombinedOutput() if err != nil { _, err := os.Stat("/proc/fs/xfs") @@ -91,17 +86,25 @@ func WrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *test } } - assert.NilError(t, err, "mount failed: %s", out) + if err != nil { + t.Fatalf("assertion failed: error is not nil: %v: mount failed: %s", err, out) + } defer func() { - assert.NilError(t, unix.Unmount(mountPoint, 0)) + if err := unix.Unmount(mountPoint, 0); err != nil { + t.Fatalf("assertion failed: error is not nil: %v", err) + } }() backingFsDev, err := makeBackingFsDev(mountPoint) - assert.NilError(t, err) + if err != nil { + t.Fatalf("assertion failed: error is not nil: %v", err) + } testDir, err := os.MkdirTemp(mountPoint, "per-test") - assert.NilError(t, err) + if err != nil { + t.Fatalf("assertion failed: error is not nil: %v", err) + } defer os.RemoveAll(testDir) testFunc(t, mountPoint, backingFsDev, testDir) @@ -113,10 +116,14 @@ func WrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *test func WrapQuotaTest(testFunc func(t *testing.T, ctrl *Control, mountPoint, testDir, testSubDir string)) func(t *testing.T, mountPoint, backingFsDev, testDir string) { return func(t *testing.T, mountPoint, backingFsDev, testDir string) { ctrl, err := NewControl(testDir) - assert.NilError(t, err) + if err != nil { + t.Fatalf("assertion failed: error is not nil: %v", err) + } testSubDir, err := os.MkdirTemp(testDir, "quota-test") - assert.NilError(t, err) + if err != nil { + t.Fatalf("assertion failed: error is not nil: %v", err) + } testFunc(t, ctrl, mountPoint, testDir, testSubDir) } } From 5dded3340cdce67139040683a6138608474e86ab Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Mon, 17 Jul 2023 14:15:57 +0100 Subject: [PATCH 175/293] ci: extract buildkit version correctly with replace-d modules Signed-off-by: Justin Chadwell (cherry picked from commit f8c0d92a22bad004cb9cbb4db704495527521c42) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/buildkit.yml | 2 +- hack/buildkit-ref | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 1a27364a7193b..161216d9836fe 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -84,7 +84,7 @@ jobs: - name: BuildKit ref run: | - echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV + echo "$(./hack/buildkit-ref)" >> $GITHUB_ENV working-directory: moby - name: Checkout BuildKit ${{ env.BUILDKIT_REF }} diff --git a/hack/buildkit-ref b/hack/buildkit-ref index 6f497c29aeb8a..b5991ad04ed8e 100755 --- a/hack/buildkit-ref +++ b/hack/buildkit-ref @@ -10,7 +10,10 @@ if [ -n "$BUILDKIT_REF" ]; then fi # get buildkit version from vendor.mod -BUILDKIT_REF=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{.Version}}' "github.com/${BUILDKIT_REPO}") +BUILDKIT_REF=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' "github.com/${BUILDKIT_REPO}") +BUILDKIT_REPO=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{if .Replace}}{{.Replace.Path}}{{else}}{{.Path}}{{end}}' "github.com/${BUILDKIT_REPO}") +BUILDKIT_REPO=${BUILDKIT_REPO#github.com/} + if [[ "${BUILDKIT_REF}" == *-*-* ]]; then # if pseudo-version, figure out just the uncommon sha (https://github.com/golang/go/issues/34745) BUILDKIT_REF=$(echo "${BUILDKIT_REF}" | awk -F"-" '{print $NF}' | awk 'BEGIN{FIELDWIDTHS="7"} {print $1}') @@ -18,4 +21,5 @@ if [[ "${BUILDKIT_REF}" == *-*-* ]]; then BUILDKIT_REF=$(curl -s "https://api.github.com/repos/${BUILDKIT_REPO}/commits/${BUILDKIT_REF}" | jq -r .sha) fi -echo "$BUILDKIT_REF" +echo "BUILDKIT_REPO=$BUILDKIT_REPO" +echo "BUILDKIT_REF=$BUILDKIT_REF" From 572de8764e57c0eb7c505a74227501e2442f7bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Tue, 18 Jul 2023 12:33:17 +0200 Subject: [PATCH 176/293] c8d/inspect: Don't duplicate digested ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If image name is already an untagged digested reference, don't produce additional digested ref. Signed-off-by: Paweł Gronowski (cherry picked from commit 028eab9ebb1e896c915db4826f2914d7f5c672e2) Signed-off-by: Paweł Gronowski --- daemon/containerd/image.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index 26dd20de29c95..ebc280994f6a8 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -130,7 +130,7 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima return nil, err } - // Each image will result in 2 references (named and digested). + // Usually each image will result in 2 references (named and digested). refs := make([]reference.Named, 0, len(tagged)*2) for _, i := range tagged { if i.UpdatedAt.After(lastUpdated) { @@ -155,6 +155,11 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima } refs = append(refs, name) + if _, ok := name.(reference.Digested); ok { + // Image name already contains a digest, so no need to create a digested reference. + continue + } + digested, err := reference.WithDigest(reference.TrimNamed(name), desc.Target.Digest) if err != nil { // This could only happen if digest is invalid, but considering that From 3c5c192bafe13589bc291fd188e275e2feea84b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Tue, 18 Jul 2023 14:21:00 +0200 Subject: [PATCH 177/293] c8d/resolveImage: Fix Digested and Named reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When resolving a reference that is both a Named and Digested, it could be resolved to an image that has the same digest, but completely different repository name. Signed-off-by: Paweł Gronowski (cherry picked from commit 48fc306764fc5c39d4284021520a0337ef7e0cb0) Signed-off-by: Paweł Gronowski --- daemon/containerd/image.go | 18 ++++++++++++++++ integration/image/remove_test.go | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index 26dd20de29c95..589ca306195d4 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -274,6 +274,24 @@ func (i *ImageService) resolveImage(ctx context.Context, refOrID string) (contai return containerdimages.Image{}, images.ErrImageDoesNotExist{Ref: parsed} } + // If reference is both Named and Digested, make sure we don't match + // images with a different repository even if digest matches. + // For example, busybox@sha256:abcdef..., shouldn't match asdf@sha256:abcdef... + if parsedNamed, ok := parsed.(reference.Named); ok { + for _, img := range imgs { + imgNamed, err := reference.ParseNormalizedNamed(img.Name) + if err != nil { + logrus.WithError(err).WithField("image", img.Name).Warn("image with invalid name encountered") + continue + } + + if parsedNamed.Name() == imgNamed.Name() { + return img, nil + } + } + return containerdimages.Image{}, images.ErrImageDoesNotExist{Ref: parsed} + } + return imgs[0], nil } diff --git a/integration/image/remove_test.go b/integration/image/remove_test.go index 11d8141da38ac..64013134279a7 100644 --- a/integration/image/remove_test.go +++ b/integration/image/remove_test.go @@ -6,9 +6,11 @@ import ( "testing" "github.com/docker/docker/api/types" + "github.com/docker/docker/errdefs" "github.com/docker/docker/integration/internal/container" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" + "gotest.tools/v3/skip" ) func TestRemoveImageOrphaning(t *testing.T) { @@ -57,3 +59,36 @@ func TestRemoveImageOrphaning(t *testing.T) { _, _, err = client.ImageInspectWithRaw(ctx, commitResp2.ID) assert.Check(t, is.ErrorContains(err, "No such image:")) } + +func TestRemoveByDigest(t *testing.T) { + skip.If(t, !testEnv.UsingSnapshotter(), "RepoDigests doesn't include tags when using graphdrivers") + + defer setupTest(t)() + ctx := context.Background() + client := testEnv.APIClient() + + err := client.ImageTag(ctx, "busybox", "test-remove-by-digest:latest") + assert.NilError(t, err) + + inspect, _, err := client.ImageInspectWithRaw(ctx, "test-remove-by-digest") + assert.NilError(t, err) + + id := "" + for _, ref := range inspect.RepoDigests { + if strings.Contains(ref, "test-remove-by-digest") { + id = ref + break + } + } + assert.Assert(t, id != "") + + t.Logf("removing %s", id) + _, err = client.ImageRemove(ctx, id, types.ImageRemoveOptions{}) + assert.NilError(t, err) + + inspect, _, err = client.ImageInspectWithRaw(ctx, "busybox") + assert.Check(t, err, "busybox image got deleted") + + inspect, _, err = client.ImageInspectWithRaw(ctx, "test-remove-by-digest") + assert.Check(t, is.ErrorType(err, errdefs.IsNotFound)) +} From 4c29864b02bf440c39ed31d2db47d8022dc6fad4 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Tue, 18 Jul 2023 08:23:39 -0600 Subject: [PATCH 178/293] hack/with-go-mod.sh: warn on stderr Signed-off-by: Bjorn Neergaard (cherry picked from commit 48ff8a95cc53ba1315705da360317e7fdd4970a7) Signed-off-by: Bjorn Neergaard --- hack/with-go-mod.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/with-go-mod.sh b/hack/with-go-mod.sh index 3be44d11ec8f4..e4210f73c479a 100755 --- a/hack/with-go-mod.sh +++ b/hack/with-go-mod.sh @@ -14,8 +14,10 @@ ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" if test -e "${ROOTDIR}/go.mod"; then { scriptname=$(basename "$0") - echo "${scriptname}: WARN: go.mod exists in the repository root!" - echo "${scriptname}: WARN: Using your go.mod instead of our generated version -- this may misbehave!" + cat >&2 <<- EOF + $scriptname: WARN: go.mod exists in the repository root! + $scriptname: WARN: Using your go.mod instead of our generated version -- this may misbehave! + EOF } >&2 else set -x From a936ae7e989cca8a6d1feaa698283ee0cfbb7875 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Tue, 18 Jul 2023 08:12:02 -0600 Subject: [PATCH 179/293] ci(buildkit): remove misleading code from buildkit-ref Post-f8c0d92a22bad004cb9cbb4db704495527521c42, BUILDKIT_REPO doesn't really do what it claims to. Instead, don't allow overloading since the import path for BuildKit is always the same, and make clear the provenance of values when generating the final variable definitions. We also better document the script, and follow some best practices for both POSIX sh and Bash. Signed-off-by: Bjorn Neergaard (cherry picked from commit 4ecc01f3ad56e608d32d141e8fc67565c195ec4e) Signed-off-by: Bjorn Neergaard --- .github/workflows/buildkit.yml | 2 -- hack/buildkit-ref | 28 +++++++++++++++++----------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 161216d9836fe..3e065c7a00636 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -49,8 +49,6 @@ jobs: test: runs-on: ubuntu-20.04 timeout-minutes: 120 - env: - BUILDKIT_REPO: moby/buildkit needs: - build strategy: diff --git a/hack/buildkit-ref b/hack/buildkit-ref index b5991ad04ed8e..99d3cf84017d7 100755 --- a/hack/buildkit-ref +++ b/hack/buildkit-ref @@ -1,8 +1,12 @@ #!/usr/bin/env bash -# This script returns the current BuildKit ref being used in moby. +# This script returns the current BuildKit ref and source repository being used. +# This script will only work with a BuildKit repository hosted on GitHub. +# If BUILDKIT_REF is already set in the environment, it will be returned as-is. +# +# The output of this script may be valid shell script, but is intended for use with +# GitHub Actions' $GITHUB_ENV. -: "${BUILDKIT_REPO=moby/buildkit}" -: "${BUILDKIT_REF=}" +buildkit_pkg=github.com/moby/buildkit if [ -n "$BUILDKIT_REF" ]; then echo "$BUILDKIT_REF" @@ -10,16 +14,18 @@ if [ -n "$BUILDKIT_REF" ]; then fi # get buildkit version from vendor.mod -BUILDKIT_REF=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' "github.com/${BUILDKIT_REPO}") -BUILDKIT_REPO=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{if .Replace}}{{.Replace.Path}}{{else}}{{.Path}}{{end}}' "github.com/${BUILDKIT_REPO}") -BUILDKIT_REPO=${BUILDKIT_REPO#github.com/} +buildkit_ref=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' "$buildkit_pkg") +buildkit_repo=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{if .Replace}}{{.Replace.Path}}{{else}}{{.Path}}{{end}}' "$buildkit_pkg") +buildkit_repo=${buildkit_repo#github.com/} -if [[ "${BUILDKIT_REF}" == *-*-* ]]; then +if [[ "${buildkit_ref}" == *-*-* ]]; then # if pseudo-version, figure out just the uncommon sha (https://github.com/golang/go/issues/34745) - BUILDKIT_REF=$(echo "${BUILDKIT_REF}" | awk -F"-" '{print $NF}' | awk 'BEGIN{FIELDWIDTHS="7"} {print $1}') + buildkit_ref=$(awk -F"-" '{print $NF}' <<< "$buildkit_ref" | awk 'BEGIN{FIELDWIDTHS="7"} {print $1}') # use github api to return full sha to be able to use it as ref - BUILDKIT_REF=$(curl -s "https://api.github.com/repos/${BUILDKIT_REPO}/commits/${BUILDKIT_REF}" | jq -r .sha) + buildkit_ref=$(curl -s "https://api.github.com/repos/${buildkit_repo}/commits/${buildkit_ref}" | jq -r .sha) fi -echo "BUILDKIT_REPO=$BUILDKIT_REPO" -echo "BUILDKIT_REF=$BUILDKIT_REF" +cat << EOF +BUILDKIT_REPO=$buildkit_repo +BUILDKIT_REF=$buildkit_ref +EOF From ff0144de3b9319acd0691bf6aa2769c9ff44cdf9 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Tue, 18 Jul 2023 08:32:44 -0600 Subject: [PATCH 180/293] ci(buildkit): remove early-return from buildkit-ref This doesn't really make sense now that this script returns a $GITHUB_ENV snippet. Signed-off-by: Bjorn Neergaard (cherry picked from commit 7310a7cd0ce75b104295905aad341bb50bc40a71) Signed-off-by: Bjorn Neergaard --- hack/buildkit-ref | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hack/buildkit-ref b/hack/buildkit-ref index 99d3cf84017d7..280bb0e25c118 100755 --- a/hack/buildkit-ref +++ b/hack/buildkit-ref @@ -1,18 +1,12 @@ #!/usr/bin/env bash # This script returns the current BuildKit ref and source repository being used. # This script will only work with a BuildKit repository hosted on GitHub. -# If BUILDKIT_REF is already set in the environment, it will be returned as-is. # # The output of this script may be valid shell script, but is intended for use with # GitHub Actions' $GITHUB_ENV. buildkit_pkg=github.com/moby/buildkit -if [ -n "$BUILDKIT_REF" ]; then - echo "$BUILDKIT_REF" - exit 0 -fi - # get buildkit version from vendor.mod buildkit_ref=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' "$buildkit_pkg") buildkit_repo=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{if .Replace}}{{.Replace.Path}}{{else}}{{.Path}}{{end}}' "$buildkit_pkg") From 1be48ec55328f9ca4cbc8682dc136ac9ffb89089 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Thu, 13 Jul 2023 18:50:03 +0200 Subject: [PATCH 181/293] c8d: Make sure the content isn't removed while we export This change add leases for all the content that will be exported, once the image(s) are exported the lease is removed, thus letting containerd's GC to do its job if needed. This fixes the case where someone would remove an image that is still being exported. This fixes the TestAPIImagesSaveAndLoad cli integration test. Signed-off-by: Djordje Lukic (cherry picked from commit f3a6b0fd08830fd0788553373b58a1008bb1bde2) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/image_exporter.go | 42 ++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/daemon/containerd/image_exporter.go b/daemon/containerd/image_exporter.go index 2eec15c7b8005..21bbcfc48ef4c 100644 --- a/daemon/containerd/image_exporter.go +++ b/daemon/containerd/image_exporter.go @@ -6,9 +6,11 @@ import ( "io" "github.com/containerd/containerd" + "github.com/containerd/containerd/content" cerrdefs "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" "github.com/containerd/containerd/images/archive" + "github.com/containerd/containerd/leases" "github.com/containerd/containerd/mount" cplatforms "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" @@ -57,11 +59,17 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea archive.WithPlatform(platform), } - ctx, release, err := i.client.WithLease(ctx) + contentStore := i.client.ContentStore() + leasesManager := i.client.LeasesService() + lease, err := leasesManager.Create(ctx, leases.WithRandomID()) if err != nil { return errdefs.System(err) } - defer release(ctx) + defer func() { + if err := leasesManager.Delete(ctx, lease); err != nil { + logrus.WithError(err).Warn("cleaning up lease") + } + }() for _, name := range names { target, err := i.resolveDescriptor(ctx, name) @@ -69,6 +77,10 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea return err } + if err = leaseContent(ctx, contentStore, leasesManager, lease, target); err != nil { + return err + } + // We may not have locally all the platforms that are specified in the index. // Export only those manifests that we have. // TODO(vvoland): Reconsider this when `--platform` is added. @@ -100,6 +112,30 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea return i.client.Export(ctx, outStream, opts...) } +// leaseContent will add a resource to the lease for each child of the descriptor making sure that it and +// its children won't be deleted while the lease exists +func leaseContent(ctx context.Context, store content.Store, leasesManager leases.Manager, lease leases.Lease, desc ocispec.Descriptor) error { + return containerdimages.Walk(ctx, containerdimages.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + _, err := store.Info(ctx, desc.Digest) + if err != nil { + if errors.Is(err, cerrdefs.ErrNotFound) { + return nil, nil + } + return nil, errdefs.System(err) + } + + r := leases.Resource{ + ID: desc.Digest.String(), + Type: "content", + } + if err := leasesManager.AddResource(ctx, lease, r); err != nil { + return nil, errdefs.System(err) + } + + return containerdimages.Children(ctx, store, desc) + }), desc) +} + // LoadImage uploads a set of images into the repository. This is the // complement of ExportImage. The input stream is an uncompressed tar // ball containing images and metadata. @@ -110,7 +146,7 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt // Create an additional image with dangling name for imported images... containerd.WithDigestRef(danglingImageName), - /// ... but only if they don't have a name or it's invalid. + // / ... but only if they don't have a name or it's invalid. containerd.WithSkipDigestRef(func(nameFromArchive string) bool { if nameFromArchive == "" { return false From aab94fb3404923ebc5c421e2b869e1d26c53fb91 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 19 Jul 2023 12:59:01 +0200 Subject: [PATCH 182/293] Dockerfile: update buildx to v0.11.2 Update the BUILDX_VERSION :) release notes: - https://github.com/docker/buildx/releases/tag/v0.11.1 - https://github.com/docker/buildx/releases/tag/v0.11.2 full diff: https://github.com/docker/buildx/compare/v0.11.0...v0.11.2 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit d78893921aac0556ec050172c7f915a3750741b1) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c8f56b6f7758a..5ef06ece05ec0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG DOCKERCLI_VERSION=v24.0.2 # cli version used for integration-cli tests ARG DOCKERCLI_INTEGRATION_REPOSITORY="https://github.com/docker/cli.git" ARG DOCKERCLI_INTEGRATION_VERSION=v17.06.2-ce -ARG BUILDX_VERSION=0.11.0 +ARG BUILDX_VERSION=0.11.2 ARG SYSTEMD="false" ARG DEBIAN_FRONTEND=noninteractive From 98a6422cbcb0ba92e4bc62ff01825008fdc9979e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 19 Jul 2023 13:06:11 +0200 Subject: [PATCH 183/293] c8d/inspect: Include platform Variant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variant was mistakenly omitted in the returned V1Image. Signed-off-by: Paweł Gronowski (cherry picked from commit 2659f7f740d7037e793a821f03d6722bc4add906) Signed-off-by: Paweł Gronowski --- daemon/containerd/image.go | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index 2ecfb7273808f..7725476d1fbd1 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -102,6 +102,7 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima OS: ociimage.OS, Architecture: ociimage.Architecture, Created: derefTimeSafely(ociimage.Created), + Variant: ociimage.Variant, Config: &containertypes.Config{ Entrypoint: ociimage.Config.Entrypoint, Env: ociimage.Config.Env, From f022632503d1e58e8008ba38397f0bdea36376f5 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 19 Jul 2023 18:20:34 +0200 Subject: [PATCH 184/293] Dockerfile: update runc binary to v1.1.8 release notes: https://github.com/opencontainers/runc/releases/tag/v1.1.8 full diff: https://github.com/opencontainers/runc/compare/v1.1.7...v1.1.9 This is the eighth patch release of the 1.1.z release branch of runc. The most notable change is the addition of RISC-V support, along with a few bug fixes. - Support riscv64. - init: do not print environment variable value. - libct: fix a race with systemd removal. - tests/int: increase num retries for oom tests. - man/runc: fixes. - Fix tmpfs mode opts when dir already exists. - docs/systemd: fix a broken link. - ci/cirrus: enable some rootless tests on cs9. - runc delete: call systemd's reset-failed. - libct/cg/sd/v1: do not update non-frozen cgroup after frozen failed. - CI: bump Fedora, Vagrant, bats. - .codespellrc: update for 2.2.5. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit df86d855f57bd28163f156941e52d6182167ef40) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- hack/dockerfile/install/runc.installer | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ef06ece05ec0..5413b699b9dc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -280,7 +280,7 @@ RUN git init . && git remote add origin "https://github.com/opencontainers/runc. # that is used. If you need to update runc, open a pull request in the containerd # project first, and update both after that is merged. When updating RUNC_VERSION, # consider updating runc in vendor.mod accordingly. -ARG RUNC_VERSION=v1.1.7 +ARG RUNC_VERSION=v1.1.8 RUN git fetch -q --depth 1 origin "${RUNC_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD FROM base AS runc-build diff --git a/hack/dockerfile/install/runc.installer b/hack/dockerfile/install/runc.installer index a58846c06f39e..289a0ca9ff478 100755 --- a/hack/dockerfile/install/runc.installer +++ b/hack/dockerfile/install/runc.installer @@ -9,7 +9,7 @@ set -e # the containerd project first, and update both after that is merged. # # When updating RUNC_VERSION, consider updating runc in vendor.mod accordingly -: "${RUNC_VERSION:=v1.1.7}" +: "${RUNC_VERSION:=v1.1.8}" install_runc() { RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp"}" From 907f83860382cf2edb2d39fc9766e714cdabaf35 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 16 May 2023 15:37:49 -0700 Subject: [PATCH 185/293] Remove Upstart and cgroups bits from Debian sysvinit script Upstart has been EOL for 8 years and isn't used by any distributions we support any more. Additionally, this removes the "cgroups v1" setup code because it's more reasonable now for us to expect something _else_ to have set up cgroups appropriately (especially cgroups v2). Signed-off-by: Tianon Gravi (cherry picked from commit ae737656f9817fbd5afab96aa083754cfb81aab0) Signed-off-by: Bjorn Neergaard --- contrib/init/sysvinit-debian/docker | 39 ----------------------------- 1 file changed, 39 deletions(-) diff --git a/contrib/init/sysvinit-debian/docker b/contrib/init/sysvinit-debian/docker index 90dbe3c956665..24aa2ca99e8a6 100755 --- a/contrib/init/sysvinit-debian/docker +++ b/contrib/init/sysvinit-debian/docker @@ -44,14 +44,6 @@ if [ ! -x $DOCKERD ]; then exit 1 fi -check_init() { - # see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it directly) - if [ -x /sbin/initctl ] && /sbin/initctl version 2> /dev/null | grep -q upstart; then - log_failure_msg "$DOCKER_DESC is managed via upstart, try using service $BASE $1" - exit 1 - fi -} - fail_unless_root() { if [ "$(id -u)" != '0' ]; then log_failure_msg "$DOCKER_DESC must be run as root" @@ -59,37 +51,10 @@ fail_unless_root() { fi } -cgroupfs_mount() { - # see also https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount - if grep -v '^#' /etc/fstab | grep -q cgroup \ - || [ ! -e /proc/cgroups ] \ - || [ ! -d /sys/fs/cgroup ]; then - return - fi - if ! mountpoint -q /sys/fs/cgroup; then - mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup - fi - ( - cd /sys/fs/cgroup - for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do - mkdir -p $sys - if ! mountpoint -q $sys; then - if ! mount -n -t cgroup -o $sys cgroup $sys; then - rmdir $sys || true - fi - fi - done - ) -} - case "$1" in start) - check_init - fail_unless_root - cgroupfs_mount - touch "$DOCKER_LOGFILE" chgrp docker "$DOCKER_LOGFILE" @@ -117,7 +82,6 @@ case "$1" in ;; stop) - check_init fail_unless_root if [ -f "$DOCKER_SSD_PIDFILE" ]; then log_begin_msg "Stopping $DOCKER_DESC: $BASE" @@ -129,7 +93,6 @@ case "$1" in ;; restart) - check_init fail_unless_root docker_pid=$(cat "$DOCKER_SSD_PIDFILE" 2> /dev/null || true) [ -n "$docker_pid" ] \ @@ -139,13 +102,11 @@ case "$1" in ;; force-reload) - check_init fail_unless_root $0 restart ;; status) - check_init status_of_proc -p "$DOCKER_SSD_PIDFILE" "$DOCKERD" "$DOCKER_DESC" ;; From eede7f09c7d03cdcfdf89853e7483b5c4d51d5c8 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 17 May 2023 14:20:14 -0700 Subject: [PATCH 186/293] Remove Upstart scripts Upstart has been EOL for 8 years and isn't used by any distributions we support any more. Signed-off-by: Tianon Gravi (cherry picked from commit 0d8087fbbc9daf521eca6f9f3ea91607f0022346) Signed-off-by: Bjorn Neergaard --- contrib/init/sysvinit-debian/docker.default | 2 +- contrib/init/upstart/docker.conf | 72 --------------------- 2 files changed, 1 insertion(+), 73 deletions(-) delete mode 100644 contrib/init/upstart/docker.conf diff --git a/contrib/init/sysvinit-debian/docker.default b/contrib/init/sysvinit-debian/docker.default index c4e93199b41df..60136c04f530a 100644 --- a/contrib/init/sysvinit-debian/docker.default +++ b/contrib/init/sysvinit-debian/docker.default @@ -1,4 +1,4 @@ -# Docker Upstart and SysVinit configuration file +# Docker SysVinit configuration file # # THIS FILE DOES NOT APPLY TO SYSTEMD diff --git a/contrib/init/upstart/docker.conf b/contrib/init/upstart/docker.conf deleted file mode 100644 index d58f7d6ac8eb0..0000000000000 --- a/contrib/init/upstart/docker.conf +++ /dev/null @@ -1,72 +0,0 @@ -description "Docker daemon" - -start on (filesystem and net-device-up IFACE!=lo) -stop on runlevel [!2345] - -limit nofile 524288 1048576 - -# Having non-zero limits causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -limit nproc unlimited unlimited - -respawn - -kill timeout 20 - -pre-start script - # see also https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount - if grep -v '^#' /etc/fstab | grep -q cgroup \ - || [ ! -e /proc/cgroups ] \ - || [ ! -d /sys/fs/cgroup ]; then - exit 0 - fi - if ! mountpoint -q /sys/fs/cgroup; then - mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup - fi - ( - cd /sys/fs/cgroup - for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do - mkdir -p $sys - if ! mountpoint -q $sys; then - if ! mount -n -t cgroup -o $sys cgroup $sys; then - rmdir $sys || true - fi - fi - done - ) -end script - -script - # modify these in /etc/default/$UPSTART_JOB (/etc/default/docker) - DOCKERD=/usr/bin/dockerd - DOCKER_OPTS= - if [ -f /etc/default/$UPSTART_JOB ]; then - . /etc/default/$UPSTART_JOB - fi - exec "$DOCKERD" $DOCKER_OPTS --raw-logs -end script - -# Don't emit "started" event until docker.sock is ready. -# See https://github.com/docker/docker/issues/6647 -post-start script - DOCKER_OPTS= - DOCKER_SOCKET= - if [ -f /etc/default/$UPSTART_JOB ]; then - . /etc/default/$UPSTART_JOB - fi - - if ! printf "%s" "$DOCKER_OPTS" | grep -qE -e '-H|--host'; then - DOCKER_SOCKET=/var/run/docker.sock - else - DOCKER_SOCKET=$(printf "%s" "$DOCKER_OPTS" | grep -oP -e '(-H|--host)\W*unix://\K(\S+)' | sed 1q) - fi - - if [ -n "$DOCKER_SOCKET" ]; then - while ! [ -e "$DOCKER_SOCKET" ]; do - initctl status $UPSTART_JOB | grep -qE "(stop|respawn)/" && exit 1 - echo "Waiting for $DOCKER_SOCKET" - sleep 0.1 - done - echo "$DOCKER_SOCKET is up" - fi -end script From fed26d5b3c9fa45afcf818f29ff30db8a16dd14c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 15 Jul 2023 20:42:29 +0200 Subject: [PATCH 187/293] vendor: github.com/moby/buildkit v0.11.7-dev full diff: https://github.com/moby/buildkit/compare/0a15675913b7...616c3f613b54a893df758428c51fad63ae2ccb7d Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 0ec73a789298d9102e71e29d4128cf02cb80b62e) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/buildkit.yml | 3 +-- builder/builder-next/worker/worker.go | 2 +- vendor.mod | 2 +- vendor.sum | 4 ++-- vendor/modules.txt | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 3e065c7a00636..7de6e0c22f941 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -13,8 +13,7 @@ on: pull_request: env: - # FIXME(thaJeztah): update to newer go versions once BuildKit's vendoring has the fix from https://github.com/moby/moby/pull/45942 - GO_VERSION: "1.20.5" + GO_VERSION: "1.20.6" DESTDIR: ./build jobs: diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index c0715d2564c9f..d91573c93f140 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -50,7 +50,7 @@ import ( ) func init() { - version.Version = "v0.11.6+0a15675913b7" + version.Version = "v0.11.6+616c3f613b54" } const labelCreatedAt = "buildkit/createdat" diff --git a/vendor.mod b/vendor.mod index d395f067930b3..0dac12a6799a9 100644 --- a/vendor.mod +++ b/vendor.mod @@ -56,7 +56,7 @@ require ( github.com/klauspost/compress v1.16.3 github.com/miekg/dns v1.1.43 github.com/mistifyio/go-zfs/v3 v3.0.1 - github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go + github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go github.com/moby/ipvs v1.1.0 github.com/moby/locker v1.0.1 github.com/moby/patternmatcher v0.5.0 diff --git a/vendor.sum b/vendor.sum index 4c9e9c9e08690..8dfeb13326506 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1043,8 +1043,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ= -github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 h1:9gjbrmALOUAJCqWL4RTwydPUiepMnkc3BNbBFiFiBeU= -github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7/go.mod h1:GCqKfHhz+pddzfgaR7WmHVEE3nKKZMMDPpK8mh3ZLv4= +github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 h1:LSh03Csyx/zQq8MreC9MYMQE/+5EkohwZMvXSS6kMZo= +github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54/go.mod h1:bMQDryngJKGvJ/ZuRFhrejurbvYSv3NkGCheQ59X4AM= github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= diff --git a/vendor/modules.txt b/vendor/modules.txt index 769853c517e9a..14e492185792b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -576,7 +576,7 @@ github.com/mistifyio/go-zfs/v3 # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 -# github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 +# github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 ## explicit; go 1.18 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types From 35a8b00b187e43bda066045c4ca95d4ff594fe4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 24 Jul 2023 15:20:31 +0200 Subject: [PATCH 188/293] hack/integration: Add TEST_INTEGRATION_FAIL_FAST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this change, integration test would fail fast and not execute all test suites when one suite fails. Change this behavior into opt-in enabled by TEST_INTEGRATION_FAIL_FAST variable. Signed-off-by: Paweł Gronowski (cherry picked from commit 48cc28e4efc21be1d288c33a9e58417df2d28306) Signed-off-by: Paweł Gronowski --- Makefile | 1 + hack/make/.integration-test-helpers | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index aa1745369969e..ff7f713466de7 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,7 @@ DOCKER_ENVS := \ -e TEST_FORCE_VALIDATE \ -e TEST_INTEGRATION_DIR \ -e TEST_INTEGRATION_USE_SNAPSHOTTER \ + -e TEST_INTEGRATION_FAIL_FAST \ -e TEST_SKIP_INTEGRATION \ -e TEST_SKIP_INTEGRATION_CLI \ -e TESTCOVERAGE \ diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index b1c31b0b9062c..177a2ec8808be 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -66,6 +66,7 @@ run_test_integration() { run_test_integration_suites() { local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS" local dirs="$1" + local failed=0 for dir in ${dirs}; do if ! ( cd "$dir" @@ -96,8 +97,16 @@ run_test_integration_suites() { --junitfile="${ABS_DEST}/${pkgname//./-}-junit-report.xml" \ --raw-command \ -- go tool test2json -p "${pkgname}" -t ./test.main ${pkgtestflags} - ); then exit 1; fi + ); then + if [ -n "${TEST_INTEGRATION_FAIL_FAST}" ]; then + exit 1 + fi + failed=1 + fi done + if [ $failed -eq 1 ]; then + exit 1 + fi } build_test_suite_binaries() { From 3029f554cc26bff91504d6c4528179e0bc01eb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 13:28:18 +0200 Subject: [PATCH 189/293] c8d/readConfig: Translate c8d NotFound to errdefs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 7379d18018255069b03273589ad481b9a54d010b) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_list.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/daemon/containerd/image_list.go b/daemon/containerd/image_list.go index e8afa43945a46..319467a449fa6 100644 --- a/daemon/containerd/image_list.go +++ b/daemon/containerd/image_list.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" timetypes "github.com/docker/docker/api/types/time" + "github.com/docker/docker/errdefs" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -479,11 +480,20 @@ func computeSharedSize(chainIDs []digest.Digest, layers map[digest.Digest]int, s func readConfig(ctx context.Context, store content.Provider, desc ocispec.Descriptor, out interface{}) error { data, err := content.ReadBlob(ctx, store, desc) if err != nil { - return errors.Wrapf(err, "failed to read config content") + err = errors.Wrapf(err, "failed to read config content") + if cerrdefs.IsNotFound(err) { + return errdefs.NotFound(err) + } + return err } + err = json.Unmarshal(data, out) if err != nil { - return errors.Wrapf(err, "could not deserialize image config") + err = errors.Wrapf(err, "could not deserialize image config") + if cerrdefs.IsNotFound(err) { + return errdefs.NotFound(err) + } + return err } return nil From fcb68e55fa4ff6503d222483af41097f0c513595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 13:22:18 +0200 Subject: [PATCH 190/293] daemon/list: Replace ErrImageDoesNotExist check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check for generic `errdefs.NotFound` rather than specific error helper struct when checking if the error is caused by the image not being present. It still works for `ErrImageDoesNotExist` because it implements the NotFound errdefs interface too. Signed-off-by: Paweł Gronowski (cherry picked from commit 5a39bee63562490ba6f95c13fcac5fe1d46e330a) Signed-off-by: Paweł Gronowski --- daemon/list.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/daemon/list.go b/daemon/list.go index 676de6f22231e..003503ea96ee3 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -11,7 +11,6 @@ import ( "github.com/docker/docker/api/types/filters" imagetypes "github.com/docker/docker/api/types/image" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/images" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/go-connections/nat" @@ -585,7 +584,7 @@ func (daemon *Daemon) refreshImage(ctx context.Context, s *container.Snapshot, f tmpImage := s.Image // keep the original ref if still valid (hasn't changed) if tmpImage != s.ImageID { img, err := daemon.imageService.GetImage(ctx, tmpImage, imagetypes.GetImageOpts{}) - if _, isDNE := err.(images.ErrImageDoesNotExist); err != nil && !isDNE { + if err != nil && !errdefs.IsNotFound(err) { return nil, err } if err != nil || img.ImageID() != s.ImageID { From 6c4121a943eed9c42d051ae94d21419c9021a0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 13:20:20 +0200 Subject: [PATCH 191/293] daemon/list: Refactor refreshImage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add context comments and make it a bit more readable. Signed-off-by: Paweł Gronowski (cherry picked from commit 68991ae240ce56e302cc9297fcc4fa04d37f7c34) Signed-off-by: Paweł Gronowski --- daemon/list.go | 71 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/daemon/list.go b/daemon/list.go index 003503ea96ee3..5b90fbbede837 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -578,21 +578,70 @@ func includeContainerInList(container *container.Snapshot, filter *listContext) return includeContainer } -// refreshImage checks if the Image ref still points to the correct ID, and updates the ref to the actual ID when it doesn't +// refreshImage checks if the Image ref still points to the correct ID, and +// updates the ref to the actual ID when it doesn't. +// This happens when the image with a reference that was used to create +// container was deleted or updated and now resolves to a different ID. +// +// For example: +// $ docker run -d busybox:latest +// $ docker ps -a +// CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +// b0318bca5aef busybox "sh" 4 seconds ago Exited (0) 3 seconds ago ecstatic_beaver +// +// After some time, busybox image got updated on the Docker Hub: +// $ docker pull busybox:latest +// +// So now busybox:latest points to a different digest, but that doesn't impact +// the ecstatic_beaver container which was still created under an older +// version. In this case, it should still point to the original image ID it was +// created from. +// +// $ docker ps -a +// CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +// b0318bca5aef 3fbc63216742 "sh" 3 years ago Exited (0) 3 years ago ecstatic_beaver func (daemon *Daemon) refreshImage(ctx context.Context, s *container.Snapshot, filter *listContext) (*types.Container, error) { c := s.Container - tmpImage := s.Image // keep the original ref if still valid (hasn't changed) - if tmpImage != s.ImageID { - img, err := daemon.imageService.GetImage(ctx, tmpImage, imagetypes.GetImageOpts{}) - if err != nil && !errdefs.IsNotFound(err) { - return nil, err - } - if err != nil || img.ImageID() != s.ImageID { - // ref changed, we need to use original ID - tmpImage = s.ImageID + + // s.Image is the image reference passed by the user to create an image + // can be a: + // - name (like nginx, ubuntu:latest, docker.io/library/busybox:latest), + // - truncated ID (abcdef), + // - full digest (sha256:abcdef...) + // + // s.ImageID is the ID of the image that s.Image resolved to at the time + // of the container creation. It's always a full digest. + + // If these match, there's nothing to refresh. + if s.Image == s.ImageID { + return &c, nil + } + + // Check if the image reference still resolves to the same digest. + img, err := daemon.imageService.GetImage(ctx, s.Image, imagetypes.GetImageOpts{}) + + // If the image is no longer found or can't be resolved for some other + // reason. Update the Image to the specific ID of the original image it + // resolved to when the container was created. + if err != nil { + if !errdefs.IsNotFound(err) { + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "containerID": c.ID, + "image": s.Image, + "imageID": s.ImageID, + }).Warn("failed to resolve container image") } + c.Image = s.ImageID + return &c, nil } - c.Image = tmpImage + + // Also update the image to the specific image ID, if the Image now + // resolves to a different ID. + if img.ImageID() != s.ImageID { + c.Image = s.ImageID + } + return &c, nil } From 45ba926c6d1d26992971e4b215719e3ead369b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 14:52:11 +0200 Subject: [PATCH 192/293] daemon/list: Drop unused arg from containerReducer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refreshImage is the only function used as a reducer and it doesn't use the `filter *listContext`. Signed-off-by: Paweł Gronowski (cherry picked from commit 13180c1c4924ac2c8529507cf2405e3afe0a2413) Signed-off-by: Paweł Gronowski --- daemon/list.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/daemon/list.go b/daemon/list.go index 5b90fbbede837..5057e188d8840 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -41,7 +41,7 @@ type iterationAction int // containerReducer represents a reducer for a container. // Returns the object to serialize by the api. -type containerReducer func(context.Context, *container.Snapshot, *listContext) (*types.Container, error) +type containerReducer func(context.Context, *container.Snapshot) (*types.Container, error) const ( // includeContainer is the action to include a container in the reducer. @@ -230,7 +230,7 @@ func (daemon *Daemon) reducePsContainer(ctx context.Context, container *containe } // transform internal container struct into api structs - newC, err := reducer(ctx, container, filter) + newC, err := reducer(ctx, container) if err != nil { return nil, err } @@ -600,7 +600,7 @@ func includeContainerInList(container *container.Snapshot, filter *listContext) // $ docker ps -a // CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES // b0318bca5aef 3fbc63216742 "sh" 3 years ago Exited (0) 3 years ago ecstatic_beaver -func (daemon *Daemon) refreshImage(ctx context.Context, s *container.Snapshot, filter *listContext) (*types.Container, error) { +func (daemon *Daemon) refreshImage(ctx context.Context, s *container.Snapshot) (*types.Container, error) { c := s.Container // s.Image is the image reference passed by the user to create an image From 7927cae910adac0b821964a2c9d3784b10e5e2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 12:09:27 +0200 Subject: [PATCH 193/293] c8d/container: Follow snapshot parents for size calculation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor GetContainerLayerSize to calculate unpacked image size only by following the snapshot parent tree directly instead of following it by using diff ids from image config. This works even if the original manifest/config used to create that container is no longer present in the content store. Signed-off-by: Paweł Gronowski (cherry picked from commit 4d8e3f54cc92dced080ff0b4daad23329cc25cc4) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_list.go | 52 ++++++------------- daemon/containerd/image_snapshot.go | 48 ++++++++++++++++++ daemon/containerd/service.go | 78 ++++++----------------------- 3 files changed, 79 insertions(+), 99 deletions(-) diff --git a/daemon/containerd/image_list.go b/daemon/containerd/image_list.go index e8afa43945a46..069a63ee56f38 100644 --- a/daemon/containerd/image_list.go +++ b/daemon/containerd/image_list.go @@ -10,6 +10,7 @@ import ( cerrdefs "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/labels" + "github.com/containerd/containerd/snapshots" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" @@ -142,41 +143,32 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions) func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, image *ImageManifest) (*types.ImageSummary, []digest.Digest, error) { diffIDs, err := image.RootFS(ctx) if err != nil { - return nil, nil, err - } - chainIDs := identity.ChainIDs(diffIDs) - - size, err := image.Size(ctx) - if err != nil { - return nil, nil, err + return nil, nil, errors.Wrapf(err, "failed to get rootfs of image %s", image.Name()) } // TODO(thaJeztah): do we need to take multiple snapshotters into account? See https://github.com/moby/moby/issues/45273 snapshotter := i.client.SnapshotService(i.snapshotter) - sizeCache := make(map[digest.Digest]int64) - snapshotSizeFn := func(d digest.Digest) (int64, error) { - if s, ok := sizeCache[d]; ok { - return s, nil - } - usage, err := snapshotter.Usage(ctx, d.String()) - if err != nil { - if cerrdefs.IsNotFound(err) { - return 0, nil - } - return 0, err + imageSnapshotID := identity.ChainID(diffIDs).String() + unpackedUsage, err := calculateSnapshotTotalUsage(ctx, snapshotter, imageSnapshotID) + if err != nil { + if !cerrdefs.IsNotFound(err) { + logrus.WithError(err).WithFields(logrus.Fields{ + "image": image.Name(), + "snapshotID": imageSnapshotID, + }).Warn("failed to calculate unpacked size of image") } - sizeCache[d] = usage.Size - return usage.Size, nil + unpackedUsage = snapshots.Usage{Size: 0} } - snapshotSize, err := computeSnapshotSize(chainIDs, snapshotSizeFn) + + contentSize, err := image.Size(ctx) if err != nil { return nil, nil, err } // totalSize is the size of the image's packed layers and snapshots // (unpacked layers) combined. - totalSize := size + snapshotSize + totalSize := contentSize + unpackedUsage.Size var repoTags, repoDigests []string rawImg := image.Metadata() @@ -225,7 +217,7 @@ func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore con Containers: -1, } - return summary, chainIDs, nil + return summary, identity.ChainIDs(diffIDs), nil } type imageFilterFunc func(image images.Image) bool @@ -446,20 +438,6 @@ func setupLabelFilter(store content.Store, fltrs filters.Args) (func(image image }, nil } -// computeSnapshotSize calculates the total size consumed by the snapshots -// for the given chainIDs. -func computeSnapshotSize(chainIDs []digest.Digest, sizeFn func(d digest.Digest) (int64, error)) (int64, error) { - var totalSize int64 - for _, chainID := range chainIDs { - size, err := sizeFn(chainID) - if err != nil { - return totalSize, err - } - totalSize += size - } - return totalSize, nil -} - func computeSharedSize(chainIDs []digest.Digest, layers map[digest.Digest]int, sizeFn func(d digest.Digest) (int64, error)) (int64, error) { var sharedSize int64 for _, chainID := range chainIDs { diff --git a/daemon/containerd/image_snapshot.go b/daemon/containerd/image_snapshot.go index a2152505c15c7..962c2100dc223 100644 --- a/daemon/containerd/image_snapshot.go +++ b/daemon/containerd/image_snapshot.go @@ -2,13 +2,18 @@ package containerd import ( "context" + "fmt" "github.com/containerd/containerd" + cerrdefs "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/snapshots" + "github.com/docker/docker/errdefs" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) // PrepareSnapshot prepares a snapshot from a parent image for a container @@ -67,3 +72,46 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentIma _, err = s.Prepare(ctx, id, parent) return err } + +// calculateSnapshotParentUsage returns the usage of all ancestors of the +// provided snapshot. It doesn't include the size of the snapshot itself. +func calculateSnapshotParentUsage(ctx context.Context, snapshotter snapshots.Snapshotter, snapshotID string) (snapshots.Usage, error) { + info, err := snapshotter.Stat(ctx, snapshotID) + if err != nil { + if cerrdefs.IsNotFound(err) { + return snapshots.Usage{}, errdefs.NotFound(err) + } + return snapshots.Usage{}, errdefs.System(errors.Wrapf(err, "snapshotter.Stat failed for %s", snapshotID)) + } + if info.Parent == "" { + return snapshots.Usage{}, errdefs.NotFound(fmt.Errorf("snapshot %s has no parent", snapshotID)) + } + + return calculateSnapshotTotalUsage(ctx, snapshotter, info.Parent) +} + +// calculateSnapshotTotalUsage returns the total usage of that snapshot +// including all of its ancestors. +func calculateSnapshotTotalUsage(ctx context.Context, snapshotter snapshots.Snapshotter, snapshotID string) (snapshots.Usage, error) { + var total snapshots.Usage + next := snapshotID + + for next != "" { + usage, err := snapshotter.Usage(ctx, next) + if err != nil { + if cerrdefs.IsNotFound(err) { + return total, errdefs.NotFound(errors.Wrapf(err, "non-existing ancestor of %s", snapshotID)) + } + return total, errdefs.System(errors.Wrapf(err, "snapshotter.Usage failed for %s", next)) + } + total.Size += usage.Size + total.Inodes += usage.Inodes + + info, err := snapshotter.Stat(ctx, next) + if err != nil { + return total, errdefs.System(errors.Wrapf(err, "snapshotter.Stat failed for %s", next)) + } + next = info.Parent + } + return total, nil +} diff --git a/daemon/containerd/service.go b/daemon/containerd/service.go index fce32665a845c..d68bcef91d77c 100644 --- a/daemon/containerd/service.go +++ b/daemon/containerd/service.go @@ -2,16 +2,15 @@ package containerd import ( "context" - "encoding/json" + "fmt" "sync/atomic" "github.com/containerd/containerd" - "github.com/containerd/containerd/content" + cerrdefs "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/snapshots" "github.com/docker/distribution/reference" - imagetypes "github.com/docker/docker/api/types/image" "github.com/docker/docker/container" daemonevents "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/images" @@ -20,8 +19,6 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/registry" - "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -165,72 +162,29 @@ func (i *ImageService) GetContainerLayerSize(ctx context.Context, containerID st } snapshotter := i.client.SnapshotService(ctr.Driver) - - usage, err := snapshotter.Usage(ctx, containerID) + rwLayerUsage, err := snapshotter.Usage(ctx, containerID) if err != nil { - return 0, 0, err - } - - imageManifest, err := getContainerImageManifest(ctr) - if err != nil { - // Best efforts attempt to pick an image. - // We don't have platform information at this point, so we can only - // assume that the platform matches host. - // Otherwise this will give a wrong base image size (different - // platform), but should be close enough. - mfst, err := i.GetImageManifest(ctx, ctr.Config.Image, imagetypes.GetImageOpts{}) - if err != nil { - // Log error, don't error out whole operation. - logrus.WithFields(logrus.Fields{ - logrus.ErrorKey: err, - "container": containerID, - }).Warn("empty ImageManifest, can't calculate base image size") - return usage.Size, 0, nil + if cerrdefs.IsNotFound(err) { + return 0, 0, errdefs.NotFound(fmt.Errorf("rw layer snapshot not found for container %s", containerID)) } - imageManifest = *mfst - } - cs := i.client.ContentStore() - - imageManifestBytes, err := content.ReadBlob(ctx, cs, imageManifest) - if err != nil { - return 0, 0, err - } - - var manifest ocispec.Manifest - if err := json.Unmarshal(imageManifestBytes, &manifest); err != nil { - return 0, 0, err + return 0, 0, errdefs.System(errors.Wrapf(err, "snapshotter.Usage failed for %s", containerID)) } - imageConfigBytes, err := content.ReadBlob(ctx, cs, manifest.Config) + unpackedUsage, err := calculateSnapshotParentUsage(ctx, snapshotter, containerID) if err != nil { - return 0, 0, err - } - var img ocispec.Image - if err := json.Unmarshal(imageConfigBytes, &img); err != nil { - return 0, 0, err - } - - sizeCache := make(map[digest.Digest]int64) - snapshotSizeFn := func(d digest.Digest) (int64, error) { - if s, ok := sizeCache[d]; ok { - return s, nil + if cerrdefs.IsNotFound(err) { + logrus.WithField("ctr", containerID).Warn("parent of container snapshot no longer present") + } else { + logrus.WithError(err).WithField("ctr", containerID).Warn("unexpected error when calculating usage of the parent snapshots") } - u, err := snapshotter.Usage(ctx, d.String()) - if err != nil { - return 0, err - } - sizeCache[d] = u.Size - return u.Size, nil - } - - chainIDs := identity.ChainIDs(img.RootFS.DiffIDs) - snapShotSize, err := computeSnapshotSize(chainIDs, snapshotSizeFn) - if err != nil { - return 0, 0, err } + logrus.WithFields(logrus.Fields{ + "rwLayerUsage": rwLayerUsage.Size, + "unpacked": unpackedUsage.Size, + }).Debug("GetContainerLayerSize") // TODO(thaJeztah): include content-store size for the image (similar to "GET /images/json") - return usage.Size, usage.Size + snapShotSize, nil + return rwLayerUsage.Size, rwLayerUsage.Size + unpackedUsage.Size, nil } // getContainerImageManifest safely dereferences ImageManifest. From 3a6899c6fd7b99691495cc03031a72afacf4e077 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 28 Jul 2023 13:20:34 +0200 Subject: [PATCH 194/293] update containerd binary to v1.7.2 - full diff: https://github.com/containerd/containerd/compare/v1.7.1...v1.7.2 - release notes: https://github.com/containerd/containerd/releases/tag/v1.7.2 ---- Welcome to the v1.7.2 release of containerd! The second patch release for containerd 1.7 includes enhancements to CRI sandbox mode, Windows snapshot mounting support, and CRI and container IO bug fixes. CRI/Sandbox Updates - Publish sandbox events - Make stats respect sandbox's platform Other Notable Updates - Mount snapshots on Windows - Notify readiness when registered plugins are ready - Fix `cio.Cancel()` should close pipes - CDI: Use CRI `Config.CDIDevices` field for CDI injection Signed-off-by: Sebastiaan van Stijn (cherry picked from commit a78381c399447cd3406aabe37c933e7d7ecfacab) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- Dockerfile.windows | 2 +- hack/dockerfile/install/containerd.installer | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5413b699b9dc0..d0a6ed18697b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -197,7 +197,7 @@ RUN git init . && git remote add origin "https://github.com/containerd/container # When updating the binary version you may also need to update the vendor # version to pick up bug fixes or new APIs, however, usually the Go packages # are built from a commit from the master branch. -ARG CONTAINERD_VERSION=v1.7.1 +ARG CONTAINERD_VERSION=v1.7.2 RUN git fetch -q --depth 1 origin "${CONTAINERD_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD FROM base AS containerd-build diff --git a/Dockerfile.windows b/Dockerfile.windows index 68b4b74830a49..0bd492388c45e 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -168,7 +168,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref ARG GO_VERSION=1.20.6 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 -ARG CONTAINERD_VERSION=v1.7.1 +ARG CONTAINERD_VERSION=v1.7.2 # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. diff --git a/hack/dockerfile/install/containerd.installer b/hack/dockerfile/install/containerd.installer index bfdd27612fe45..e1ebc732f771c 100755 --- a/hack/dockerfile/install/containerd.installer +++ b/hack/dockerfile/install/containerd.installer @@ -15,7 +15,7 @@ set -e # the binary version you may also need to update the vendor version to pick up # bug fixes or new APIs, however, usually the Go packages are built from a # commit from the master branch. -: "${CONTAINERD_VERSION:=v1.7.1}" +: "${CONTAINERD_VERSION:=v1.7.2}" install_containerd() ( echo "Install containerd version $CONTAINERD_VERSION" From 02241b05fcc065b654f89e8f401a56ce35beb1e0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 28 Jul 2023 13:21:20 +0200 Subject: [PATCH 195/293] update containerd binary to v1.7.3 - full diff: https://github.com/containerd/containerd/compare/v1.7.2...v1.7.3 - release notes: https://github.com/containerd/containerd/releases/tag/v1.7.3 ---- Welcome to the v1.7.3 release of containerd! The third patch release for containerd 1.7 contains various fixes and updates. Notable Updates - RunC: Update runc binary to v1.1.8 - CRI: Fix `additionalGids`: it should fallback to `imageConfig.User` when `securityContext.RunAsUser`,`RunAsUsername` are empty - CRI: write generated CNI config atomically - Port-Forward: Correctly handle known errors - Resolve docker.NewResolver race condition - Fix `net.ipv4.ping_group_range` with userns - Runtime/V2/RunC: handle early exits w/o big locks - SecComp: always allow `name_to_handle_at` - CRI: Windows Pod Stats: Add a check to skip stats for containers that are not running - Task: don't `close()` io before cancel() - Remove CNI conf_template deprecation - Fix issue for HPC pod metrics Signed-off-by: Sebastiaan van Stijn (cherry picked from commit bf48d3ec29c1b48615c3a938d0144e8367e00dd3) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- Dockerfile.windows | 2 +- hack/dockerfile/install/containerd.installer | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d0a6ed18697b5..9a8e5ea7ffaaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -197,7 +197,7 @@ RUN git init . && git remote add origin "https://github.com/containerd/container # When updating the binary version you may also need to update the vendor # version to pick up bug fixes or new APIs, however, usually the Go packages # are built from a commit from the master branch. -ARG CONTAINERD_VERSION=v1.7.2 +ARG CONTAINERD_VERSION=v1.7.3 RUN git fetch -q --depth 1 origin "${CONTAINERD_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD FROM base AS containerd-build diff --git a/Dockerfile.windows b/Dockerfile.windows index 0bd492388c45e..e98f714e16c75 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -168,7 +168,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref ARG GO_VERSION=1.20.6 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 -ARG CONTAINERD_VERSION=v1.7.2 +ARG CONTAINERD_VERSION=v1.7.3 # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. diff --git a/hack/dockerfile/install/containerd.installer b/hack/dockerfile/install/containerd.installer index e1ebc732f771c..0e2b5f88c1c9d 100755 --- a/hack/dockerfile/install/containerd.installer +++ b/hack/dockerfile/install/containerd.installer @@ -15,7 +15,7 @@ set -e # the binary version you may also need to update the vendor version to pick up # bug fixes or new APIs, however, usually the Go packages are built from a # commit from the master branch. -: "${CONTAINERD_VERSION:=v1.7.2}" +: "${CONTAINERD_VERSION:=v1.7.3}" install_containerd() ( echo "Install containerd version $CONTAINERD_VERSION" From b6568d2dd5cdbd46d70f8a20378ad6c33da9852d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 28 Jul 2023 16:27:36 +0200 Subject: [PATCH 196/293] api/types/filters: fix errors not being matched by errors.Is() I found that the errors returned weren't matched with `errors.Is()` when wrapped. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 490fee7d45dfe69d313156b17d2e4ad12c2277ba) Signed-off-by: Sebastiaan van Stijn --- api/types/filters/parse.go | 10 +++---- api/types/filters/parse_test.go | 50 ++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/api/types/filters/parse.go b/api/types/filters/parse.go index 887648cf3e3d2..0c39ab5f18b53 100644 --- a/api/types/filters/parse.go +++ b/api/types/filters/parse.go @@ -98,7 +98,7 @@ func FromJSON(p string) (Args, error) { // Fallback to parsing arguments in the legacy slice format deprecated := map[string][]string{} if legacyErr := json.Unmarshal(raw, &deprecated); legacyErr != nil { - return args, invalidFilter{} + return args, &invalidFilter{} } args.fields = deprecatedArgs(deprecated) @@ -206,7 +206,7 @@ func (args Args) GetBoolOrDefault(key string, defaultValue bool) (bool, error) { } if len(fieldValues) == 0 { - return defaultValue, invalidFilter{key, nil} + return defaultValue, &invalidFilter{key, nil} } isFalse := fieldValues["0"] || fieldValues["false"] @@ -216,7 +216,7 @@ func (args Args) GetBoolOrDefault(key string, defaultValue bool) (bool, error) { invalid := !isFalse && !isTrue if conflicting || invalid { - return defaultValue, invalidFilter{key, args.Get(key)} + return defaultValue, &invalidFilter{key, args.Get(key)} } else if isFalse { return false, nil } else if isTrue { @@ -224,7 +224,7 @@ func (args Args) GetBoolOrDefault(key string, defaultValue bool) (bool, error) { } // This code shouldn't be reached. - return defaultValue, unreachableCode{Filter: key, Value: args.Get(key)} + return defaultValue, &unreachableCode{Filter: key, Value: args.Get(key)} } // ExactMatch returns true if the source matches exactly one of the values. @@ -282,7 +282,7 @@ func (args Args) Contains(field string) bool { func (args Args) Validate(accepted map[string]bool) error { for name := range args.fields { if !accepted[name] { - return invalidFilter{name, nil} + return &invalidFilter{name, nil} } } return nil diff --git a/api/types/filters/parse_test.go b/api/types/filters/parse_test.go index 74672ea2c99a7..5b1e2de307595 100644 --- a/api/types/filters/parse_test.go +++ b/api/types/filters/parse_test.go @@ -3,6 +3,7 @@ package filters // import "github.com/docker/docker/api/types/filters" import ( "encoding/json" "errors" + "fmt" "sort" "testing" @@ -95,15 +96,19 @@ func TestFromJSON(t *testing.T) { if err == nil { t.Fatalf("Expected an error with %v, got nothing", invalid) } - var invalidFilterError invalidFilter + var invalidFilterError *invalidFilter if !errors.As(err, &invalidFilterError) { t.Fatalf("Expected an invalidFilter error, got %T", err) } + wrappedErr := fmt.Errorf("something went wrong: %w", err) + if !errors.Is(wrappedErr, err) { + t.Errorf("Expected a wrapped error to be detected as invalidFilter") + } } for expectedArgs, matchers := range valid { - for _, json := range matchers { - args, err := FromJSON(json) + for _, jsonString := range matchers { + args, err := FromJSON(jsonString) if err != nil { t.Fatal(err) } @@ -358,9 +363,13 @@ func TestValidate(t *testing.T) { if err == nil { t.Fatal("Expected to return an error, got nil") } - var invalidFilterError invalidFilter + var invalidFilterError *invalidFilter if !errors.As(err, &invalidFilterError) { - t.Fatalf("Expected an invalidFilter error, got %T", err) + t.Errorf("Expected an invalidFilter error, got %T", err) + } + wrappedErr := fmt.Errorf("something went wrong: %w", err) + if !errors.Is(wrappedErr, err) { + t.Errorf("Expected a wrapped error to be detected as invalidFilter") } } @@ -421,7 +430,7 @@ func TestClone(t *testing.T) { } func TestGetBoolOrDefault(t *testing.T) { - for _, tC := range []struct { + for _, tc := range []struct { name string args map[string][]string defValue bool @@ -452,7 +461,7 @@ func TestGetBoolOrDefault(t *testing.T) { "dangling": {"potato"}, }, defValue: true, - expectedErr: invalidFilter{Filter: "dangling", Value: []string{"potato"}}, + expectedErr: &invalidFilter{Filter: "dangling", Value: []string{"potato"}}, expectedValue: true, }, { @@ -461,7 +470,7 @@ func TestGetBoolOrDefault(t *testing.T) { "dangling": {"banana", "potato"}, }, defValue: true, - expectedErr: invalidFilter{Filter: "dangling", Value: []string{"banana", "potato"}}, + expectedErr: &invalidFilter{Filter: "dangling", Value: []string{"banana", "potato"}}, expectedValue: true, }, { @@ -470,7 +479,7 @@ func TestGetBoolOrDefault(t *testing.T) { "dangling": {"false", "true"}, }, defValue: false, - expectedErr: invalidFilter{Filter: "dangling", Value: []string{"false", "true"}}, + expectedErr: &invalidFilter{Filter: "dangling", Value: []string{"false", "true"}}, expectedValue: false, }, { @@ -479,7 +488,7 @@ func TestGetBoolOrDefault(t *testing.T) { "dangling": {"false", "true", "1"}, }, defValue: true, - expectedErr: invalidFilter{Filter: "dangling", Value: []string{"false", "true", "1"}}, + expectedErr: &invalidFilter{Filter: "dangling", Value: []string{"false", "true", "1"}}, expectedValue: true, }, { @@ -501,35 +510,38 @@ func TestGetBoolOrDefault(t *testing.T) { expectedValue: false, }, } { - tC := tC - t.Run(tC.name, func(t *testing.T) { + tc := tc + t.Run(tc.name, func(t *testing.T) { a := NewArgs() - for key, values := range tC.args { + for key, values := range tc.args { for _, value := range values { a.Add(key, value) } } - value, err := a.GetBoolOrDefault("dangling", tC.defValue) + value, err := a.GetBoolOrDefault("dangling", tc.defValue) - if tC.expectedErr == nil { + if tc.expectedErr == nil { assert.Check(t, is.Nil(err)) } else { - assert.Check(t, is.ErrorType(err, tC.expectedErr)) + assert.Check(t, is.ErrorType(err, tc.expectedErr)) // Check if error is the same. - expected := tC.expectedErr.(invalidFilter) - actual := err.(invalidFilter) + expected := tc.expectedErr.(*invalidFilter) + actual := err.(*invalidFilter) assert.Check(t, is.Equal(expected.Filter, actual.Filter)) sort.Strings(expected.Value) sort.Strings(actual.Value) assert.Check(t, is.DeepEqual(expected.Value, actual.Value)) + + wrappedErr := fmt.Errorf("something went wrong: %w", err) + assert.Check(t, errors.Is(wrappedErr, err), "Expected a wrapped error to be detected as invalidFilter") } - assert.Check(t, is.Equal(tC.expectedValue, value)) + assert.Check(t, is.Equal(tc.expectedValue, value)) }) } From 6be708aa7d02727598fed6fe73424a90d5ecd82d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 29 Jul 2023 19:07:09 +0200 Subject: [PATCH 197/293] vendor: gotest.tools/v3 v3.5.0 - go.mod: update dependencies and go version by - Use Go1.20 - Fix couple of typos - Added `WithStdout` and `WithStderr` helpers - Moved `cmdOperators` handling from `RunCmd` to `StartCmd` - Deprecate `assert.ErrorType` - Remove outdated Dockerfile - add godoc links full diff: https://github.com/gotestyourself/gotest.tools/compare/v3.4.0...v3.5.0 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ce053a14aa3fd28ae7225d67220b177ade75bc0c) Signed-off-by: Sebastiaan van Stijn --- hack/validate/golangci-lint.yml | 4 ++ vendor.mod | 2 +- vendor.sum | 4 +- vendor/gotest.tools/v3/assert/assert.go | 69 ++++++++++---------- vendor/gotest.tools/v3/assert/cmp/compare.go | 25 +++---- vendor/gotest.tools/v3/assert/cmp/result.go | 12 ++-- vendor/gotest.tools/v3/assert/opt/opt.go | 12 ++-- vendor/gotest.tools/v3/env/env.go | 2 +- vendor/gotest.tools/v3/icmd/command.go | 17 +++-- vendor/gotest.tools/v3/icmd/ops.go | 14 ++++ vendor/gotest.tools/v3/poll/check.go | 4 +- vendor/gotest.tools/v3/poll/poll.go | 18 ++--- vendor/modules.txt | 4 +- 13 files changed, 106 insertions(+), 81 deletions(-) diff --git a/hack/validate/golangci-lint.yml b/hack/validate/golangci-lint.yml index ab188068506df..2832ad7e77352 100644 --- a/hack/validate/golangci-lint.yml +++ b/hack/validate/golangci-lint.yml @@ -126,6 +126,10 @@ issues: - text: "SA1019: httputil.ErrPersistEOF" linters: - staticcheck + # FIXME temporarily suppress these (see https://github.com/gotestyourself/gotest.tools/issues/272) + - text: "SA1019: (assert|cmp|is)\\.ErrorType is deprecated" + linters: + - staticcheck # Maximum issues count per one linter. Set to 0 to disable. Default is 50. max-issues-per-linter: 0 diff --git a/vendor.mod b/vendor.mod index 0dac12a6799a9..93b49543cb836 100644 --- a/vendor.mod +++ b/vendor.mod @@ -94,7 +94,7 @@ require ( golang.org/x/time v0.3.0 google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 google.golang.org/grpc v1.50.1 - gotest.tools/v3 v3.4.0 + gotest.tools/v3 v3.5.0 resenje.org/singleflight v0.3.0 ) diff --git a/vendor.sum b/vendor.sum index 8dfeb13326506..7a265df2c3284 100644 --- a/vendor.sum +++ b/vendor.sum @@ -2223,8 +2223,8 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= -gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/gotest.tools/v3/assert/assert.go b/vendor/gotest.tools/v3/assert/assert.go index f75f9f510e6b9..c418bd07b5c43 100644 --- a/vendor/gotest.tools/v3/assert/assert.go +++ b/vendor/gotest.tools/v3/assert/assert.go @@ -4,7 +4,7 @@ values in tests. When an assertion fails a helpful error message is printed. # Example usage -All the assertions in this package use testing.T.Helper to mark themselves as +All the assertions in this package use [testing.T.Helper] to mark themselves as test helpers. This allows the testing package to print the filename and line number of the file function that failed. @@ -67,19 +67,19 @@ message is omitted from these examples for brevity. # Assert and Check -Assert and Check are very similar, they both accept a Comparison, and fail +[Assert] and [Check] are very similar, they both accept a [cmp.Comparison], and fail the test when the comparison fails. The one difference is that Assert uses -testing.T.FailNow to fail the test, which will end the test execution immediately. -Check uses testing.T.Fail to fail the test, which allows it to return the +[testing.T.FailNow] to fail the test, which will end the test execution immediately. +Check uses [testing.T.Fail] to fail the test, which allows it to return the result of the comparison, then proceed with the rest of the test case. -Like testing.T.FailNow, Assert must be called from the goroutine running the test, -not from other goroutines created during the test. Check is safe to use from any +Like [testing.T.FailNow], [Assert] must be called from the goroutine running the test, +not from other goroutines created during the test. [Check] is safe to use from any goroutine. # Comparisons -Package http://pkg.go.dev/gotest.tools/v3/assert/cmp provides +Package [gotest.tools/v3/assert/cmp] provides many common comparisons. Additional comparisons can be written to compare values in other ways. See the example Assert (CustomComparison). @@ -98,11 +98,11 @@ import ( "gotest.tools/v3/internal/assert" ) -// BoolOrComparison can be a bool, cmp.Comparison, or error. See Assert for +// BoolOrComparison can be a bool, [cmp.Comparison], or error. See [Assert] for // details about how this type is used. type BoolOrComparison interface{} -// TestingT is the subset of testing.T used by the assert package. +// TestingT is the subset of [testing.T] (see also [testing.TB]) used by the assert package. type TestingT interface { FailNow() Fail() @@ -133,11 +133,11 @@ type helperT interface { // // Extra details can be added to the failure message using msgAndArgs. msgAndArgs // may be either a single string, or a format string and args that will be -// passed to fmt.Sprintf. +// passed to [fmt.Sprintf]. // -// Assert uses t.FailNow to fail the test. Like t.FailNow, Assert must be called +// Assert uses [testing.TB.FailNow] to fail the test. Like t.FailNow, Assert must be called // from the goroutine running the test function, not from other -// goroutines created during the test. Use Check from other goroutines. +// goroutines created during the test. Use [Check] from other goroutines. func Assert(t TestingT, comparison BoolOrComparison, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { ht.Helper() @@ -151,7 +151,7 @@ func Assert(t TestingT, comparison BoolOrComparison, msgAndArgs ...interface{}) // failed, a failure message is printed, and Check returns false. If the comparison // is successful Check returns true. Check may be called from any goroutine. // -// See Assert for details about the comparison arg and failure messages. +// See [Assert] for details about the comparison arg and failure messages. func Check(t TestingT, comparison BoolOrComparison, msgAndArgs ...interface{}) bool { if ht, ok := t.(helperT); ok { ht.Helper() @@ -166,9 +166,9 @@ func Check(t TestingT, comparison BoolOrComparison, msgAndArgs ...interface{}) b // NilError fails the test immediately if err is not nil, and includes err.Error // in the failure message. // -// NilError uses t.FailNow to fail the test. Like t.FailNow, NilError must be +// NilError uses [testing.TB.FailNow] to fail the test. Like t.FailNow, NilError must be // called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check from other goroutines. +// goroutines created during the test. Use [Check] from other goroutines. func NilError(t TestingT, err error, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { ht.Helper() @@ -193,9 +193,9 @@ func NilError(t TestingT, err error, msgAndArgs ...interface{}) { // the unified diff will be augmented by replacing whitespace characters with // visible characters to identify the whitespace difference. // -// Equal uses t.FailNow to fail the test. Like t.FailNow, Equal must be +// Equal uses [testing.T.FailNow] to fail the test. Like t.FailNow, Equal must be // called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.Equal from other +// goroutines created during the test. Use [Check] with [cmp.Equal] from other // goroutines. func Equal(t TestingT, x, y interface{}, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { @@ -206,15 +206,15 @@ func Equal(t TestingT, x, y interface{}, msgAndArgs ...interface{}) { } } -// DeepEqual uses google/go-cmp (https://godoc.org/github.com/google/go-cmp/cmp) +// DeepEqual uses [github.com/google/go-cmp/cmp] // to assert two values are equal and fails the test if they are not equal. // -// Package http://pkg.go.dev/gotest.tools/v3/assert/opt provides some additional +// Package [gotest.tools/v3/assert/opt] provides some additional // commonly used Options. // -// DeepEqual uses t.FailNow to fail the test. Like t.FailNow, DeepEqual must be +// DeepEqual uses [testing.T.FailNow] to fail the test. Like t.FailNow, DeepEqual must be // called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.DeepEqual from other +// goroutines created during the test. Use [Check] with [cmp.DeepEqual] from other // goroutines. func DeepEqual(t TestingT, x, y interface{}, opts ...gocmp.Option) { if ht, ok := t.(helperT); ok { @@ -227,13 +227,13 @@ func DeepEqual(t TestingT, x, y interface{}, opts ...gocmp.Option) { // Error fails the test if err is nil, or if err.Error is not equal to expected. // Both err.Error and expected will be included in the failure message. -// Error performs an exact match of the error text. Use ErrorContains if only -// part of the error message is relevant. Use ErrorType or ErrorIs to compare +// Error performs an exact match of the error text. Use [ErrorContains] if only +// part of the error message is relevant. Use [ErrorType] or [ErrorIs] to compare // errors by type. // -// Error uses t.FailNow to fail the test. Like t.FailNow, Error must be +// Error uses [testing.T.FailNow] to fail the test. Like t.FailNow, Error must be // called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.Error from other +// goroutines created during the test. Use [Check] with [cmp.Error] from other // goroutines. func Error(t TestingT, err error, expected string, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { @@ -248,9 +248,9 @@ func Error(t TestingT, err error, expected string, msgAndArgs ...interface{}) { // contain the expected substring. Both err.Error and the expected substring // will be included in the failure message. // -// ErrorContains uses t.FailNow to fail the test. Like t.FailNow, ErrorContains +// ErrorContains uses [testing.T.FailNow] to fail the test. Like t.FailNow, ErrorContains // must be called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.ErrorContains from other +// goroutines created during the test. Use [Check] with [cmp.ErrorContains] from other // goroutines. func ErrorContains(t TestingT, err error, substring string, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { @@ -262,8 +262,7 @@ func ErrorContains(t TestingT, err error, substring string, msgAndArgs ...interf } // ErrorType fails the test if err is nil, or err is not the expected type. -// Most new code should use ErrorIs instead. ErrorType may be deprecated in the -// future. +// New code should use ErrorIs instead. // // Expected can be one of: // @@ -281,10 +280,12 @@ func ErrorContains(t TestingT, err error, substring string, msgAndArgs ...interf // reflect.Type // The assertion fails if err does not implement the reflect.Type. // -// ErrorType uses t.FailNow to fail the test. Like t.FailNow, ErrorType +// ErrorType uses [testing.T.FailNow] to fail the test. Like t.FailNow, ErrorType // must be called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.ErrorType from other +// goroutines created during the test. Use [Check] with [cmp.ErrorType] from other // goroutines. +// +// Deprecated: Use [ErrorIs] func ErrorType(t TestingT, err error, expected interface{}, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { ht.Helper() @@ -295,12 +296,12 @@ func ErrorType(t TestingT, err error, expected interface{}, msgAndArgs ...interf } // ErrorIs fails the test if err is nil, or the error does not match expected -// when compared using errors.Is. See https://golang.org/pkg/errors/#Is for +// when compared using errors.Is. See [errors.Is] for // accepted arguments. // -// ErrorIs uses t.FailNow to fail the test. Like t.FailNow, ErrorIs +// ErrorIs uses [testing.T.FailNow] to fail the test. Like t.FailNow, ErrorIs // must be called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.ErrorIs from other +// goroutines created during the test. Use [Check] with [cmp.ErrorIs] from other // goroutines. func ErrorIs(t TestingT, err error, expected error, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { diff --git a/vendor/gotest.tools/v3/assert/cmp/compare.go b/vendor/gotest.tools/v3/assert/cmp/compare.go index 4112b00433f22..118844f35d071 100644 --- a/vendor/gotest.tools/v3/assert/cmp/compare.go +++ b/vendor/gotest.tools/v3/assert/cmp/compare.go @@ -12,17 +12,16 @@ import ( "gotest.tools/v3/internal/format" ) -// Comparison is a function which compares values and returns ResultSuccess if +// Comparison is a function which compares values and returns [ResultSuccess] if // the actual value matches the expected value. If the values do not match the -// Result will contain a message about why it failed. +// [Result] will contain a message about why it failed. type Comparison func() Result -// DeepEqual compares two values using google/go-cmp -// (https://godoc.org/github.com/google/go-cmp/cmp) +// DeepEqual compares two values using [github.com/google/go-cmp/cmp] // and succeeds if the values are equal. // // The comparison can be customized using comparison Options. -// Package http://pkg.go.dev/gotest.tools/v3/assert/opt provides some additional +// Package [gotest.tools/v3/assert/opt] provides some additional // commonly used Options. func DeepEqual(x, y interface{}, opts ...cmp.Option) Comparison { return func() (result Result) { @@ -61,7 +60,7 @@ func toResult(success bool, msg string) Result { return ResultFailure(msg) } -// RegexOrPattern may be either a *regexp.Regexp or a string that is a valid +// RegexOrPattern may be either a [*regexp.Regexp] or a string that is a valid // regexp pattern. type RegexOrPattern interface{} @@ -95,7 +94,7 @@ func Regexp(re RegexOrPattern, v string) Comparison { } } -// Equal succeeds if x == y. See assert.Equal for full documentation. +// Equal succeeds if x == y. See [gotest.tools/v3/assert.Equal] for full documentation. func Equal(x, y interface{}) Comparison { return func() Result { switch { @@ -159,10 +158,10 @@ func Len(seq interface{}, expected int) Comparison { // slice, or array. // // If collection is a string, item must also be a string, and is compared using -// strings.Contains(). +// [strings.Contains]. // If collection is a Map, contains will succeed if item is a key in the map. // If collection is a slice or array, item is compared to each item in the -// sequence using reflect.DeepEqual(). +// sequence using [reflect.DeepEqual]. func Contains(collection interface{}, item interface{}) Comparison { return func() Result { colValue := reflect.ValueOf(collection) @@ -259,7 +258,7 @@ func formatErrorMessage(err error) string { // Nil succeeds if obj is a nil interface, pointer, or function. // -// Use NilError() for comparing errors. Use Len(obj, 0) for comparing slices, +// Use [gotest.tools/v3/assert.NilError] for comparing errors. Use Len(obj, 0) for comparing slices, // maps, and channels. func Nil(obj interface{}) Comparison { msgFunc := func(value reflect.Value) string { @@ -306,7 +305,9 @@ func isNil(obj interface{}, msgFunc func(reflect.Value) string) Comparison { // // reflect.Type // -// Fails if err does not implement the reflect.Type +// Fails if err does not implement the [reflect.Type]. +// +// Deprecated: Use [ErrorIs] func ErrorType(err error, expected interface{}) Comparison { return func() Result { switch expectedType := expected.(type) { @@ -381,7 +382,7 @@ var ( ) // ErrorIs succeeds if errors.Is(actual, expected) returns true. See -// https://golang.org/pkg/errors/#Is for accepted argument values. +// [errors.Is] for accepted argument values. func ErrorIs(actual error, expected error) Comparison { return func() Result { if errors.Is(actual, expected) { diff --git a/vendor/gotest.tools/v3/assert/cmp/result.go b/vendor/gotest.tools/v3/assert/cmp/result.go index 28ef8d3d46c5d..9992ede54465b 100644 --- a/vendor/gotest.tools/v3/assert/cmp/result.go +++ b/vendor/gotest.tools/v3/assert/cmp/result.go @@ -10,12 +10,12 @@ import ( "gotest.tools/v3/internal/source" ) -// A Result of a Comparison. +// A Result of a [Comparison]. type Result interface { Success() bool } -// StringResult is an implementation of Result that reports the error message +// StringResult is an implementation of [Result] that reports the error message // string verbatim and does not provide any templating or formatting of the // message. type StringResult struct { @@ -34,16 +34,16 @@ func (r StringResult) FailureMessage() string { return r.message } -// ResultSuccess is a constant which is returned by a ComparisonWithResult to +// ResultSuccess is a constant which is returned by a [Comparison] to // indicate success. var ResultSuccess = StringResult{success: true} -// ResultFailure returns a failed Result with a failure message. +// ResultFailure returns a failed [Result] with a failure message. func ResultFailure(message string) StringResult { return StringResult{message: message} } -// ResultFromError returns ResultSuccess if err is nil. Otherwise ResultFailure +// ResultFromError returns [ResultSuccess] if err is nil. Otherwise [ResultFailure] // is returned with the error message as the failure message. func ResultFromError(err error) Result { if err == nil { @@ -74,7 +74,7 @@ func (r templatedResult) UpdatedExpected(stackIndex int) error { return source.UpdateExpectedValue(stackIndex+1, r.data["x"], r.data["y"]) } -// ResultFailureTemplate returns a Result with a template string and data which +// ResultFailureTemplate returns a [Result] with a template string and data which // can be used to format a failure message. The template may access data from .Data, // the comparison args with the callArg function, and the formatNode function may // be used to format the call args. diff --git a/vendor/gotest.tools/v3/assert/opt/opt.go b/vendor/gotest.tools/v3/assert/opt/opt.go index 357cdf2ebae1f..bd4c9dc3a2e8f 100644 --- a/vendor/gotest.tools/v3/assert/opt/opt.go +++ b/vendor/gotest.tools/v3/assert/opt/opt.go @@ -11,7 +11,7 @@ import ( gocmp "github.com/google/go-cmp/cmp" ) -// DurationWithThreshold returns a gocmp.Comparer for comparing time.Duration. The +// DurationWithThreshold returns a [gocmp.Comparer] for comparing [time.Duration]. The // Comparer returns true if the difference between the two Duration values is // within the threshold and neither value is zero. func DurationWithThreshold(threshold time.Duration) gocmp.Option { @@ -28,7 +28,7 @@ func cmpDuration(threshold time.Duration) func(x, y time.Duration) bool { } } -// TimeWithThreshold returns a gocmp.Comparer for comparing time.Time. The +// TimeWithThreshold returns a [gocmp.Comparer] for comparing [time.Time]. The // Comparer returns true if the difference between the two Time values is // within the threshold and neither value is zero. func TimeWithThreshold(threshold time.Duration) gocmp.Option { @@ -45,12 +45,12 @@ func cmpTime(threshold time.Duration) func(x, y time.Time) bool { } } -// PathString is a gocmp.FilterPath filter that returns true when path.String() +// PathString is a [gocmp.FilterPath] filter that returns true when path.String() // matches any of the specs. // // The path spec is a dot separated string where each segment is a field name. // Slices, Arrays, and Maps are always matched against every element in the -// sequence. gocmp.Indirect, gocmp.Transform, and gocmp.TypeAssertion are always +// sequence. [gocmp.Indirect], [gocmp.Transform], and [gocmp.TypeAssertion] are always // ignored. // // Note: this path filter is not type safe. Incorrect paths will be silently @@ -66,7 +66,7 @@ func PathString(specs ...string) func(path gocmp.Path) bool { } } -// PathDebug is a gocmp.FilerPath filter that always returns false. It prints +// PathDebug is a [gocmp.FilterPath] filter that always returns false. It prints // each path it receives. It can be used to debug path matching problems. func PathDebug(path gocmp.Path) bool { fmt.Printf("PATH string=%s gostring=%s\n", path, path.GoString()) @@ -95,7 +95,7 @@ func stepTypeFields(step gocmp.PathStep) string { return "" } -// PathField is a gocmp.FilerPath filter that matches a struct field by name. +// PathField is a [gocmp.FilterPath] filter that matches a struct field by name. // PathField will match every instance of the field in a recursive or nested // structure. func PathField(structType interface{}, field string) func(gocmp.Path) bool { diff --git a/vendor/gotest.tools/v3/env/env.go b/vendor/gotest.tools/v3/env/env.go index 71efc39307a57..9653cf1875edc 100644 --- a/vendor/gotest.tools/v3/env/env.go +++ b/vendor/gotest.tools/v3/env/env.go @@ -72,7 +72,7 @@ func PatchAll(t assert.TestingT, env map[string]string) func() { return clean } -// ToMap takes a list of strings in the format returned by os.Environ() and +// ToMap takes a list of strings in the format returned by [os.Environ] and // returns a mapping of keys to values. func ToMap(env []string) map[string]string { result := map[string]string{} diff --git a/vendor/gotest.tools/v3/icmd/command.go b/vendor/gotest.tools/v3/icmd/command.go index a15834bab408e..a3e167a013348 100644 --- a/vendor/gotest.tools/v3/icmd/command.go +++ b/vendor/gotest.tools/v3/icmd/command.go @@ -195,6 +195,7 @@ type Cmd struct { Timeout time.Duration Stdin io.Reader Stdout io.Writer + Stderr io.Writer Dir string Env []string ExtraFiles []*os.File @@ -207,10 +208,7 @@ func Command(command string, args ...string) Cmd { // RunCmd runs a command and returns a Result func RunCmd(cmd Cmd, cmdOperators ...CmdOp) *Result { - for _, op := range cmdOperators { - op(&cmd) - } - result := StartCmd(cmd) + result := StartCmd(cmd, cmdOperators...) if result.Error != nil { return result } @@ -223,7 +221,10 @@ func RunCommand(command string, args ...string) *Result { } // StartCmd starts a command, but doesn't wait for it to finish -func StartCmd(cmd Cmd) *Result { +func StartCmd(cmd Cmd, cmdOperators ...CmdOp) *Result { + for _, op := range cmdOperators { + op(&cmd) + } result := buildCmd(cmd) if result.Error != nil { return result @@ -252,7 +253,11 @@ func buildCmd(cmd Cmd) *Result { } else { execCmd.Stdout = outBuffer } - execCmd.Stderr = errBuffer + if cmd.Stderr != nil { + execCmd.Stderr = io.MultiWriter(errBuffer, cmd.Stderr) + } else { + execCmd.Stderr = errBuffer + } execCmd.ExtraFiles = cmd.ExtraFiles return &Result{ diff --git a/vendor/gotest.tools/v3/icmd/ops.go b/vendor/gotest.tools/v3/icmd/ops.go index 35c3958d52bb3..aa3bc1e8f8eec 100644 --- a/vendor/gotest.tools/v3/icmd/ops.go +++ b/vendor/gotest.tools/v3/icmd/ops.go @@ -38,6 +38,20 @@ func WithStdin(r io.Reader) CmdOp { } } +// WithStdout sets the standard output of the command to the specified writer +func WithStdout(w io.Writer) CmdOp { + return func(c *Cmd) { + c.Stdout = w + } +} + +// WithStderr sets the standard error of the command to the specified writer +func WithStderr(w io.Writer) CmdOp { + return func(c *Cmd) { + c.Stderr = w + } +} + // WithExtraFile adds a file descriptor to the command func WithExtraFile(f *os.File) CmdOp { return func(c *Cmd) { diff --git a/vendor/gotest.tools/v3/poll/check.go b/vendor/gotest.tools/v3/poll/check.go index 46880f5b25cae..fa0f21c1e17c8 100644 --- a/vendor/gotest.tools/v3/poll/check.go +++ b/vendor/gotest.tools/v3/poll/check.go @@ -5,7 +5,7 @@ import ( "os" ) -// Check is a function which will be used as check for the WaitOn method. +// Check is a function which will be used as check for the [WaitOn] method. type Check func(t LogT) Result // FileExists looks on filesystem and check that path exists. @@ -29,7 +29,7 @@ func FileExists(path string) Check { } // Connection try to open a connection to the address on the -// named network. See net.Dial for a description of the network and +// named network. See [net.Dial] for a description of the network and // address parameters. func Connection(network, address string) Check { return func(t LogT) Result { diff --git a/vendor/gotest.tools/v3/poll/poll.go b/vendor/gotest.tools/v3/poll/poll.go index 29c5b40e187fd..cfd6d43ace742 100644 --- a/vendor/gotest.tools/v3/poll/poll.go +++ b/vendor/gotest.tools/v3/poll/poll.go @@ -11,13 +11,13 @@ import ( "gotest.tools/v3/internal/assert" ) -// TestingT is the subset of testing.T used by WaitOn +// TestingT is the subset of [testing.T] used by [WaitOn] type TestingT interface { LogT Fatalf(format string, args ...interface{}) } -// LogT is a logging interface that is passed to the WaitOn check function +// LogT is a logging interface that is passed to the [WaitOn] check function type LogT interface { Log(args ...interface{}) Logf(format string, args ...interface{}) @@ -27,7 +27,7 @@ type helperT interface { Helper() } -// Settings are used to configure the behaviour of WaitOn +// Settings are used to configure the behaviour of [WaitOn] type Settings struct { // Timeout is the maximum time to wait for the condition. Defaults to 10s. Timeout time.Duration @@ -57,7 +57,7 @@ func WithTimeout(timeout time.Duration) SettingOp { } } -// Result of a check performed by WaitOn +// Result of a check performed by [WaitOn] type Result interface { // Error indicates that the check failed and polling should stop, and the // the has failed @@ -86,20 +86,20 @@ func (r result) Error() error { return r.err } -// Continue returns a Result that indicates to WaitOn that it should continue +// Continue returns a [Result] that indicates to [WaitOn] that it should continue // polling. The message text will be used as the failure message if the timeout // is reached. func Continue(message string, args ...interface{}) Result { return result{message: fmt.Sprintf(message, args...)} } -// Success returns a Result where Done() returns true, which indicates to WaitOn +// Success returns a [Result] where Done() returns true, which indicates to [WaitOn] // that it should stop polling and exit without an error. func Success() Result { return result{done: true} } -// Error returns a Result that indicates to WaitOn that it should fail the test +// Error returns a [Result] that indicates to [WaitOn] that it should fail the test // and stop polling. func Error(err error) Result { return result{err: err} @@ -143,9 +143,9 @@ func WaitOn(t TestingT, check Check, pollOps ...SettingOp) { } } -// Compare values using the cmp.Comparison. If the comparison fails return a +// Compare values using the [cmp.Comparison]. If the comparison fails return a // result which indicates to WaitOn that it should continue waiting. -// If the comparison is successful then WaitOn stops polling. +// If the comparison is successful then [WaitOn] stops polling. func Compare(compare cmp.Comparison) Result { buf := new(logBuffer) if assert.RunComparison(buf, assert.ArgsAtZeroIndex, compare) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 14e492185792b..65ad85f86de6e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1284,8 +1284,8 @@ google.golang.org/protobuf/types/known/fieldmaskpb google.golang.org/protobuf/types/known/structpb google.golang.org/protobuf/types/known/timestamppb google.golang.org/protobuf/types/known/wrapperspb -# gotest.tools/v3 v3.4.0 -## explicit; go 1.13 +# gotest.tools/v3 v3.5.0 +## explicit; go 1.17 gotest.tools/v3/assert gotest.tools/v3/assert/cmp gotest.tools/v3/assert/opt From a6f8e973422abb8dfc47be6d229798b7a75f9674 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 11 Mar 2021 18:57:01 +0000 Subject: [PATCH 198/293] Improve test daemon logging 1. On failed start tail the daemon logs 2. Exposes generic tailing functions to make test debugging simpler Signed-off-by: Brian Goff (cherry picked from commit 914888cf8bfb7a25f8e8018f753d923d3495a0b3) Signed-off-by: Sebastiaan van Stijn --- testutil/daemon/daemon.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/testutil/daemon/daemon.go b/testutil/daemon/daemon.go index 98230960c6da0..f0b5537b3089d 100644 --- a/testutil/daemon/daemon.go +++ b/testutil/daemon/daemon.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/container" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/pkg/tailfile" "github.com/docker/docker/testutil/request" "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" @@ -296,10 +297,41 @@ func (d *Daemon) Cleanup(t testing.TB) { cleanupNetworkNamespace(t, d) } +// TailLogsT attempts to tail N lines from the daemon logs. +// If there is an error the error is only logged, it does not cause an error with the test. +func (d *Daemon) TailLogsT(t LogT, n int) { + lines, err := d.TailLogs(n) + if err != nil { + t.Logf("[%s] %v", d.id, err) + return + } + for _, l := range lines { + t.Logf("[%s] %s", d.id, string(l)) + } +} + +// TailLogs tails N lines from the daemon logs +func (d *Daemon) TailLogs(n int) ([][]byte, error) { + logF, err := os.Open(d.logFile.Name()) + if err != nil { + return nil, errors.Wrap(err, "error opening daemon log file after failed start") + } + + defer logF.Close() + lines, err := tailfile.TailFile(logF, n) + if err != nil { + return nil, errors.Wrap(err, "error tailing log daemon logs") + } + + return lines, nil + +} + // Start starts the daemon and return once it is ready to receive requests. func (d *Daemon) Start(t testing.TB, args ...string) { t.Helper() if err := d.StartWithError(args...); err != nil { + d.TailLogsT(t, 20) d.DumpStackAndQuit() // in case the daemon is stuck t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err) } From d6536d44e972dd9fe9c632691a1bb7232264653b Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 28 Jul 2023 19:50:24 +0000 Subject: [PATCH 199/293] TestDaemonProxy: check proxy settings early Allows tests to report their proxy settings for easier troubleshooting on failures. Signed-off-by: Brian Goff (cherry picked from commit 8197752d681ec700ceee6f5d71f9cb1fec2adf19) Signed-off-by: Sebastiaan van Stijn --- integration/daemon/daemon_test.go | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index b6e00c81b9c04..28c5c422cd41e 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -190,6 +190,12 @@ func TestDaemonProxy(t *testing.T) { defer func() { _ = c.Close() }() ctx := context.Background() d.Start(t) + defer d.Stop(t) + + info := d.Info(t) + assert.Check(t, is.Equal(info.HTTPProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.NoProxy, "example.com")) _, err := c.ImagePull(ctx, "example.org:5000/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") @@ -199,12 +205,6 @@ func TestDaemonProxy(t *testing.T) { _, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5000", "should not have used proxy") - - info := d.Info(t) - assert.Equal(t, info.HTTPProxy, proxyServer.URL) - assert.Equal(t, info.HTTPSProxy, proxyServer.URL) - assert.Equal(t, info.NoProxy, "example.com") - d.Stop(t) }) // Configure proxy through command-line flags @@ -218,6 +218,7 @@ func TestDaemonProxy(t *testing.T) { d := daemon.New(t) d.Start(t, "--http-proxy", proxyServer.URL, "--https-proxy", proxyServer.URL, "--no-proxy", "example.com") + defer d.Stop(t) logs, err := d.ReadLogFile() assert.NilError(t, err) @@ -231,6 +232,11 @@ func TestDaemonProxy(t *testing.T) { defer func() { _ = c.Close() }() ctx := context.Background() + info := d.Info(t) + assert.Check(t, is.Equal(info.HTTPProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.NoProxy, "example.com")) + _, err = c.ImagePull(ctx, "example.org:5001/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5001") @@ -239,13 +245,6 @@ func TestDaemonProxy(t *testing.T) { _, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5001", "should not have used proxy") - - info := d.Info(t) - assert.Equal(t, info.HTTPProxy, proxyServer.URL) - assert.Equal(t, info.HTTPSProxy, proxyServer.URL) - assert.Equal(t, info.NoProxy, "example.com") - - d.Stop(t) }) // Configure proxy through configuration file @@ -267,6 +266,12 @@ func TestDaemonProxy(t *testing.T) { assert.NilError(t, os.WriteFile(configFile, []byte(configJSON), 0644)) d.Start(t, "--config-file", configFile) + defer d.Stop(t) + + info := d.Info(t) + assert.Check(t, is.Equal(info.HTTPProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.NoProxy, "example.com")) logs, err := d.ReadLogFile() assert.NilError(t, err) @@ -285,11 +290,6 @@ func TestDaemonProxy(t *testing.T) { assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5002", "should not have used proxy") - info := d.Info(t) - assert.Equal(t, info.HTTPProxy, proxyServer.URL) - assert.Equal(t, info.HTTPSProxy, proxyServer.URL) - assert.Equal(t, info.NoProxy, "example.com") - d.Stop(t) }) From a49bca97dfffb0f9bf42986721d401d58c2f76ae Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 28 Jul 2023 21:21:17 +0000 Subject: [PATCH 200/293] Fix daemon proxy test for "reload sanitized" I noticed this was always being skipped because of race conditions checking the logs. This change adds a log scanner which will look through the logs line by line rather than allocating a big buffer. Additionally it adds a `poll.Check` which we can use to actually wait for the desired log entry. Signed-off-by: Brian Goff Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 476e788090feafb246f800e71e926aa76d671fb3) Signed-off-by: Sebastiaan van Stijn --- integration/daemon/daemon_test.go | 25 ++++++++----------- testutil/daemon/daemon.go | 40 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index 28c5c422cd41e..78d13d1256c23 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -22,6 +22,7 @@ import ( "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" "gotest.tools/v3/icmd" + "gotest.tools/v3/poll" "gotest.tools/v3/skip" ) @@ -319,32 +320,26 @@ func TestDaemonProxy(t *testing.T) { // Make sure values are sanitized when reloading the daemon-config t.Run("reload sanitized", func(t *testing.T) { + t.Parallel() + + ctx := context.Background() const ( proxyRawURL = "https://" + userPass + "example.org" proxyURL = "https://xxxxx:xxxxx@example.org" ) d := daemon.New(t) - d.Start(t, "--http-proxy", proxyRawURL, "--https-proxy", proxyRawURL, "--no-proxy", "example.com") + d.Start(t, "--iptables=false", "--http-proxy", proxyRawURL, "--https-proxy", proxyRawURL, "--no-proxy", "example.com") defer d.Stop(t) err := d.Signal(syscall.SIGHUP) assert.NilError(t, err) - logs, err := d.ReadLogFile() - assert.NilError(t, err) + poll.WaitOn(t, d.PollCheckLogs(ctx, "Reloaded configuration:")) + poll.WaitOn(t, d.PollCheckLogs(ctx, proxyURL)) - // FIXME: there appears to ba a race condition, which causes ReadLogFile - // to not contain the full logs after signaling the daemon to reload, - // causing the test to fail here. As a workaround, check if we - // received the "reloaded" message after signaling, and only then - // check that it's sanitized properly. For more details on this - // issue, see https://github.com/moby/moby/pull/42835/files#r713120315 - if !strings.Contains(string(logs), "Reloaded configuration:") { - t.Skip("Skipping test, because we did not find 'Reloaded configuration' in the logs") - } - - assert.Assert(t, is.Contains(string(logs), proxyURL)) - assert.Assert(t, !strings.Contains(string(logs), userPass), "logs should not contain the non-sanitized proxy URL: %s", string(logs)) + ok, logs, err := d.ScanLogs(ctx, userPass) + assert.NilError(t, err) + assert.Assert(t, !ok, "logs should not contain the non-sanitized proxy URL: %s", logs) }) } diff --git a/testutil/daemon/daemon.go b/testutil/daemon/daemon.go index f0b5537b3089d..67f7c9297a4d2 100644 --- a/testutil/daemon/daemon.go +++ b/testutil/daemon/daemon.go @@ -1,8 +1,10 @@ package daemon // import "github.com/docker/docker/testutil/daemon" import ( + "bufio" "context" "encoding/json" + "io" "net/http" "os" "os/exec" @@ -25,6 +27,7 @@ import ( "github.com/docker/go-connections/tlsconfig" "github.com/pkg/errors" "gotest.tools/v3/assert" + "gotest.tools/v3/poll" ) // LogT is the subset of the testing.TB interface used by the daemon. @@ -310,6 +313,43 @@ func (d *Daemon) TailLogsT(t LogT, n int) { } } +// PollCheckLogs is a poll.Check that checks the daemon logs for the passed in string (`contains`). +func (d *Daemon) PollCheckLogs(ctx context.Context, contains string) poll.Check { + return func(t poll.LogT) poll.Result { + ok, _, err := d.ScanLogs(ctx, contains) + if err != nil { + return poll.Error(err) + } + if !ok { + return poll.Continue("waiting for %q in daemon logs", contains) + } + return poll.Success() + } +} + +// ScanLogs scans the daemon logs for the passed in string (`contains`). +// If the context is canceled, the function returns false but does not error out the test. +func (d *Daemon) ScanLogs(ctx context.Context, contains string) (bool, string, error) { + stat, err := d.logFile.Stat() + if err != nil { + return false, "", err + } + rdr := io.NewSectionReader(d.logFile, 0, stat.Size()) + + scanner := bufio.NewScanner(rdr) + for scanner.Scan() { + if strings.Contains(scanner.Text(), contains) { + return true, scanner.Text(), nil + } + select { + case <-ctx.Done(): + return false, "", ctx.Err() + default: + } + } + return false, "", scanner.Err() +} + // TailLogs tails N lines from the daemon logs func (d *Daemon) TailLogs(n int) ([][]byte, error) { logF, err := os.Open(d.logFile.Name()) From 4cd50eb1edc2032dc1920c2615a2ce2a80d4bf7d Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 28 Jul 2023 22:15:10 +0000 Subject: [PATCH 201/293] TestDaemonProxy: use new scanners to check logs Also fixes up some cleanup issues. Signed-off-by: Brian Goff Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 1a51898d2ea45a8f6e0d349878ea410782a24208) Signed-off-by: Sebastiaan van Stijn --- integration/daemon/daemon_test.go | 142 +++++++++++++++++------------- testutil/daemon/daemon.go | 46 ++++++++-- 2 files changed, 117 insertions(+), 71 deletions(-) diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index 78d13d1256c23..f4565c9616766 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -9,7 +9,6 @@ import ( "os/exec" "path/filepath" "runtime" - "strings" "syscall" "testing" @@ -170,27 +169,34 @@ func TestDaemonProxy(t *testing.T) { skip.If(t, runtime.GOOS == "windows", "cannot start multiple daemons on windows") skip.If(t, os.Getenv("DOCKER_ROOTLESS") != "", "cannot connect to localhost proxy in rootless environment") - var received string - proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - received = r.Host - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte("OK")) - })) - defer proxyServer.Close() + newProxy := func(rcvd *string, t *testing.T) *httptest.Server { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *rcvd = r.Host + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte("OK")) + })) + t.Cleanup(s.Close) + return s + } const userPass = "myuser:mypassword@" // Configure proxy through env-vars t.Run("environment variables", func(t *testing.T) { - t.Setenv("HTTP_PROXY", proxyServer.URL) - t.Setenv("HTTPS_PROXY", proxyServer.URL) - t.Setenv("NO_PROXY", "example.com") + t.Parallel() - d := daemon.New(t) - c := d.NewClientT(t) - defer func() { _ = c.Close() }() ctx := context.Background() - d.Start(t) + var received string + proxyServer := newProxy(&received, t) + + d := daemon.New(t, daemon.WithEnvVars( + "HTTP_PROXY="+proxyServer.URL, + "HTTPS_PROXY="+proxyServer.URL, + "NO_PROXY=example.com", + )) + c := d.NewClientT(t) + + d.Start(t, "--iptables=false") defer d.Stop(t) info := d.Info(t) @@ -210,35 +216,45 @@ func TestDaemonProxy(t *testing.T) { // Configure proxy through command-line flags t.Run("command-line options", func(t *testing.T) { - t.Setenv("HTTP_PROXY", "http://"+userPass+"from-env-http.invalid") - t.Setenv("http_proxy", "http://"+userPass+"from-env-http.invalid") - t.Setenv("HTTPS_PROXY", "https://"+userPass+"myuser:mypassword@from-env-https.invalid") - t.Setenv("https_proxy", "https://"+userPass+"myuser:mypassword@from-env-https.invalid") - t.Setenv("NO_PROXY", "ignore.invalid") - t.Setenv("no_proxy", "ignore.invalid") + t.Parallel() - d := daemon.New(t) - d.Start(t, "--http-proxy", proxyServer.URL, "--https-proxy", proxyServer.URL, "--no-proxy", "example.com") + ctx := context.Background() + var received string + proxyServer := newProxy(&received, t) + + d := daemon.New(t, daemon.WithEnvVars( + "HTTP_PROXY="+"http://"+userPass+"from-env-http.invalid", + "http_proxy="+"http://"+userPass+"from-env-http.invalid", + "HTTPS_PROXY="+"https://"+userPass+"myuser:mypassword@from-env-https-invalid", + "https_proxy="+"https://"+userPass+"myuser:mypassword@from-env-https-invalid", + "NO_PROXY=ignore.invalid", + "no_proxy=ignore.invalid", + )) + d.Start(t, "--iptables=false", "--http-proxy", proxyServer.URL, "--https-proxy", proxyServer.URL, "--no-proxy", "example.com") defer d.Stop(t) - logs, err := d.ReadLogFile() - assert.NilError(t, err) - assert.Assert(t, is.Contains(string(logs), "overriding existing proxy variable with value from configuration")) - for _, v := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} { - assert.Assert(t, is.Contains(string(logs), "name="+v)) - assert.Assert(t, !strings.Contains(string(logs), userPass), "logs should not contain the non-sanitized proxy URL: %s", string(logs)) - } - c := d.NewClientT(t) - defer func() { _ = c.Close() }() - ctx := context.Background() info := d.Info(t) assert.Check(t, is.Equal(info.HTTPProxy, proxyServer.URL)) assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) assert.Check(t, is.Equal(info.NoProxy, "example.com")) - _, err = c.ImagePull(ctx, "example.org:5001/some/image:latest", types.ImagePullOptions{}) + ok, _ := d.ScanLogsT(ctx, t, daemon.ScanLogsMatchAll( + "overriding existing proxy variable with value from configuration", + "http_proxy", + "HTTP_PROXY", + "https_proxy", + "HTTPS_PROXY", + "no_proxy", + "NO_PROXY", + )) + assert.Assert(t, ok) + + ok, logs := d.ScanLogsT(ctx, t, daemon.ScanLogsMatchString(userPass)) + assert.Assert(t, !ok, "logs should not contain the non-sanitized proxy URL: %s", logs) + + _, err := c.ImagePull(ctx, "example.org:5001/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5001") @@ -250,23 +266,27 @@ func TestDaemonProxy(t *testing.T) { // Configure proxy through configuration file t.Run("configuration file", func(t *testing.T) { - t.Setenv("HTTP_PROXY", "http://"+userPass+"from-env-http.invalid") - t.Setenv("http_proxy", "http://"+userPass+"from-env-http.invalid") - t.Setenv("HTTPS_PROXY", "https://"+userPass+"myuser:mypassword@from-env-https.invalid") - t.Setenv("https_proxy", "https://"+userPass+"myuser:mypassword@from-env-https.invalid") - t.Setenv("NO_PROXY", "ignore.invalid") - t.Setenv("no_proxy", "ignore.invalid") + t.Parallel() + ctx := context.Background() - d := daemon.New(t) + var received string + proxyServer := newProxy(&received, t) + + d := daemon.New(t, daemon.WithEnvVars( + "HTTP_PROXY="+"http://"+userPass+"from-env-http.invalid", + "http_proxy="+"http://"+userPass+"from-env-http.invalid", + "HTTPS_PROXY="+"https://"+userPass+"myuser:mypassword@from-env-https-invalid", + "https_proxy="+"https://"+userPass+"myuser:mypassword@from-env-https-invalid", + "NO_PROXY=ignore.invalid", + "no_proxy=ignore.invalid", + )) c := d.NewClientT(t) - defer func() { _ = c.Close() }() - ctx := context.Background() configFile := filepath.Join(d.RootDir(), "daemon.json") configJSON := fmt.Sprintf(`{"proxies":{"http-proxy":%[1]q, "https-proxy": %[1]q, "no-proxy": "example.com"}}`, proxyServer.URL) assert.NilError(t, os.WriteFile(configFile, []byte(configJSON), 0644)) - d.Start(t, "--config-file", configFile) + d.Start(t, "--iptables=false", "--config-file", configFile) defer d.Stop(t) info := d.Info(t) @@ -274,15 +294,17 @@ func TestDaemonProxy(t *testing.T) { assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) assert.Check(t, is.Equal(info.NoProxy, "example.com")) - logs, err := d.ReadLogFile() - assert.NilError(t, err) - assert.Assert(t, is.Contains(string(logs), "overriding existing proxy variable with value from configuration")) - for _, v := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} { - assert.Assert(t, is.Contains(string(logs), "name="+v)) - assert.Assert(t, !strings.Contains(string(logs), userPass), "logs should not contain the non-sanitized proxy URL: %s", string(logs)) - } - - _, err = c.ImagePull(ctx, "example.org:5002/some/image:latest", types.ImagePullOptions{}) + d.ScanLogsT(ctx, t, daemon.ScanLogsMatchAll( + "overriding existing proxy variable with value from configuration", + "http_proxy", + "HTTP_PROXY", + "https_proxy", + "HTTPS_PROXY", + "no_proxy", + "NO_PROXY", + )) + + _, err := c.ImagePull(ctx, "example.org:5002/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5002") @@ -290,12 +312,11 @@ func TestDaemonProxy(t *testing.T) { _, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5002", "should not have used proxy") - - d.Stop(t) }) // Conflicting options (passed both through command-line options and config file) t.Run("conflicting options", func(t *testing.T) { + ctx := context.Background() const ( proxyRawURL = "https://" + userPass + "example.org" proxyURL = "https://xxxxx:xxxxx@example.org" @@ -309,13 +330,12 @@ func TestDaemonProxy(t *testing.T) { err := d.StartWithError("--http-proxy", proxyRawURL, "--https-proxy", proxyRawURL, "--no-proxy", "example.com", "--config-file", configFile, "--validate") assert.ErrorContains(t, err, "daemon exited during startup") - logs, err := d.ReadLogFile() - assert.NilError(t, err) + expected := fmt.Sprintf( `the following directives are specified both as a flag and in the configuration file: http-proxy: (from flag: %[1]s, from file: %[1]s), https-proxy: (from flag: %[1]s, from file: %[1]s), no-proxy: (from flag: example.com, from file: example.com)`, proxyURL, ) - assert.Assert(t, is.Contains(string(logs), expected)) + poll.WaitOn(t, d.PollCheckLogs(ctx, daemon.ScanLogsMatchString(expected))) }) // Make sure values are sanitized when reloading the daemon-config @@ -334,11 +354,9 @@ func TestDaemonProxy(t *testing.T) { err := d.Signal(syscall.SIGHUP) assert.NilError(t, err) - poll.WaitOn(t, d.PollCheckLogs(ctx, "Reloaded configuration:")) - poll.WaitOn(t, d.PollCheckLogs(ctx, proxyURL)) + poll.WaitOn(t, d.PollCheckLogs(ctx, daemon.ScanLogsMatchAll("Reloaded configuration:", proxyURL))) - ok, logs, err := d.ScanLogs(ctx, userPass) - assert.NilError(t, err) + ok, logs := d.ScanLogsT(ctx, t, daemon.ScanLogsMatchString(userPass)) assert.Assert(t, !ok, "logs should not contain the non-sanitized proxy URL: %s", logs) }) } diff --git a/testutil/daemon/daemon.go b/testutil/daemon/daemon.go index 67f7c9297a4d2..360c39c89d592 100644 --- a/testutil/daemon/daemon.go +++ b/testutil/daemon/daemon.go @@ -277,6 +277,7 @@ func (d *Daemon) NewClientT(t testing.TB, extraOpts ...client.Opt) *client.Clien c, err := d.NewClient(extraOpts...) assert.NilError(t, err, "[%s] could not create daemon client", d.id) + t.Cleanup(func() { c.Close() }) return c } @@ -313,23 +314,51 @@ func (d *Daemon) TailLogsT(t LogT, n int) { } } -// PollCheckLogs is a poll.Check that checks the daemon logs for the passed in string (`contains`). -func (d *Daemon) PollCheckLogs(ctx context.Context, contains string) poll.Check { +// PollCheckLogs is a poll.Check that checks the daemon logs using the passed in match function. +func (d *Daemon) PollCheckLogs(ctx context.Context, match func(s string) bool) poll.Check { return func(t poll.LogT) poll.Result { - ok, _, err := d.ScanLogs(ctx, contains) + ok, _, err := d.ScanLogs(ctx, match) if err != nil { return poll.Error(err) } if !ok { - return poll.Continue("waiting for %q in daemon logs", contains) + return poll.Continue("waiting for daemon logs match") } return poll.Success() } } -// ScanLogs scans the daemon logs for the passed in string (`contains`). -// If the context is canceled, the function returns false but does not error out the test. -func (d *Daemon) ScanLogs(ctx context.Context, contains string) (bool, string, error) { +// ScanLogsMatchString returns a function that can be used to scan the daemon logs for the passed in string (`contains`). +func ScanLogsMatchString(contains string) func(string) bool { + return func(line string) bool { + return strings.Contains(line, contains) + } +} + +// ScanLogsMatchAll returns a function that can be used to scan the daemon logs until *all* the passed in strings are matched +func ScanLogsMatchAll(contains ...string) func(string) bool { + matched := make(map[string]bool) + return func(line string) bool { + for _, c := range contains { + if strings.Contains(line, c) { + matched[c] = true + } + } + return len(matched) == len(contains) + } +} + +// ScanLogsT uses `ScanLogs` to match the daemon logs using the passed in match function. +// If there is an error or the match fails, the test will fail. +func (d *Daemon) ScanLogsT(ctx context.Context, t testing.TB, match func(s string) bool) (bool, string) { + t.Helper() + ok, line, err := d.ScanLogs(ctx, match) + assert.NilError(t, err) + return ok, line +} + +// ScanLogs scans the daemon logs and passes each line to the match function. +func (d *Daemon) ScanLogs(ctx context.Context, match func(s string) bool) (bool, string, error) { stat, err := d.logFile.Stat() if err != nil { return false, "", err @@ -338,7 +367,7 @@ func (d *Daemon) ScanLogs(ctx context.Context, contains string) (bool, string, e scanner := bufio.NewScanner(rdr) for scanner.Scan() { - if strings.Contains(scanner.Text(), contains) { + if match(scanner.Text()) { return true, scanner.Text(), nil } select { @@ -364,7 +393,6 @@ func (d *Daemon) TailLogs(n int) ([][]byte, error) { } return lines, nil - } // Start starts the daemon and return once it is ready to receive requests. From b7c5385b81b5a7a58f6a2cc0d78d380f1fb6931c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 1 Aug 2023 23:46:01 +0200 Subject: [PATCH 202/293] update to go1.20.7 Includes a fix for CVE-2023-29409 go1.20.7 (released 2023-08-01) includes a security fix to the crypto/tls package, as well as bug fixes to the assembler and the compiler. See the Go 1.20.7 milestone on our issue tracker for details: - https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved - full diff: https://github.com/golang/go/compare/go1.20.6...go1.20.7 From the mailing list announcement: [security] Go 1.20.7 and Go 1.19.12 are released Hello gophers, We have just released Go versions 1.20.7 and 1.19.12, minor point releases. These minor releases include 1 security fixes following the security policy: - crypto/tls: restrict RSA keys in certificates to <= 8192 bits Extremely large RSA keys in certificate chains can cause a client/server to expend significant CPU time verifying signatures. Limit this by restricting the size of RSA keys transmitted during handshakes to <= 8192 bits. Based on a survey of publicly trusted RSA keys, there are currently only three certificates in circulation with keys larger than this, and all three appear to be test certificates that are not actively deployed. It is possible there are larger keys in use in private PKIs, but we target the web PKI, so causing breakage here in the interests of increasing the default safety of users of crypto/tls seems reasonable. Thanks to Mateusz Poliwczak for reporting this issue. View the release notes for more information: https://go.dev/doc/devel/release#go1.20.7 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit d5cb7cdeae32f071dfa243c2a34925a23dd50679) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/.windows.yml | 2 +- .github/workflows/buildkit.yml | 2 +- .github/workflows/test.yml | 2 +- Dockerfile | 2 +- Dockerfile.simple | 2 +- Dockerfile.windows | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml index 66812324a817d..5e8736e34de9b 100644 --- a/.github/workflows/.windows.yml +++ b/.github/workflows/.windows.yml @@ -15,7 +15,7 @@ on: default: false env: - GO_VERSION: "1.20.6" + GO_VERSION: "1.20.7" GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 WINDOWS_BASE_IMAGE: mcr.microsoft.com/windows/servercore diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 7de6e0c22f941..95910dbf3c9e2 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -13,7 +13,7 @@ on: pull_request: env: - GO_VERSION: "1.20.6" + GO_VERSION: "1.20.7" DESTDIR: ./build jobs: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d5fb2f6a8c5c..2e7467f418c75 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ on: pull_request: env: - GO_VERSION: "1.20.6" + GO_VERSION: "1.20.7" GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 ITG_CLI_MATRIX_SIZE: 6 diff --git a/Dockerfile b/Dockerfile index 9a8e5ea7ffaaa..d4cbee5021570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20.6 +ARG GO_VERSION=1.20.7 ARG BASE_DEBIAN_DISTRO="bullseye" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" ARG XX_VERSION=1.2.1 diff --git a/Dockerfile.simple b/Dockerfile.simple index 8605aa3fcd0c7..91d2eaf04d068 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -5,7 +5,7 @@ # This represents the bare minimum required to build and test Docker. -ARG GO_VERSION=1.20.6 +ARG GO_VERSION=1.20.7 ARG BASE_DEBIAN_DISTRO="bullseye" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" diff --git a/Dockerfile.windows b/Dockerfile.windows index e98f714e16c75..d3783a9331bf7 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -165,7 +165,7 @@ FROM microsoft/windowsservercore # Use PowerShell as the default shell SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -ARG GO_VERSION=1.20.6 +ARG GO_VERSION=1.20.7 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 ARG CONTAINERD_VERSION=v1.7.3 From 128838227e0cf234e2a1b4f4610189dea363d120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Tue, 8 Aug 2023 12:52:28 +0200 Subject: [PATCH 203/293] hack/test: Don't fail-fast before integration-cli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If TEST_INTEGRATION_FAIL_FAST is not set, run the integration-cli tests even if integration tests failed. Signed-off-by: Paweł Gronowski (cherry picked from commit 6841a53d1764bd65d08fe655a147ae3c24c977c1) Signed-off-by: Paweł Gronowski --- hack/make/.integration-test-helpers | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index 177a2ec8808be..e9062ac66cdec 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -55,11 +55,23 @@ fi run_test_integration() { set_platform_timeout + local failed=0 if [ -z "${TEST_SKIP_INTEGRATION}" ]; then - run_test_integration_suites "${integration_api_dirs}" + if ! run_test_integration_suites "${integration_api_dirs}"; then + if [ -n "${TEST_INTEGRATION_FAIL_FAST}" ]; then + return 1 + fi + failed=1 + fi fi if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then - TIMEOUT=360m run_test_integration_suites integration-cli + if ! TIMEOUT=360m run_test_integration_suites integration-cli; then + return 1 + fi + fi + + if [ $failed -eq 1 ]; then + return 1 fi } @@ -99,13 +111,13 @@ run_test_integration_suites() { -- go tool test2json -p "${pkgname}" -t ./test.main ${pkgtestflags} ); then if [ -n "${TEST_INTEGRATION_FAIL_FAST}" ]; then - exit 1 + return 1 fi failed=1 fi done if [ $failed -eq 1 ]; then - exit 1 + return 1 fi } From 749e687e1b52e87a51a92df02334603a68072462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Tue, 25 Apr 2023 12:00:12 +0200 Subject: [PATCH 204/293] integration/windows: Unskip some kill tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unskip: - TestKillWithStopSignalAndRestartPolicies - TestKillStoppedContainer integration tests on Windows. Signed-off-by: Paweł Gronowski (cherry picked from commit dd1c95edcde99cc5a2673567c4c47e939b8a4e41) Signed-off-by: Paweł Gronowski --- integration/container/kill_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integration/container/kill_test.go b/integration/container/kill_test.go index 42adb7867a3d0..219cd55bb2fb1 100644 --- a/integration/container/kill_test.go +++ b/integration/container/kill_test.go @@ -82,7 +82,6 @@ func TestKillContainer(t *testing.T) { } func TestKillWithStopSignalAndRestartPolicies(t *testing.T) { - skip.If(t, testEnv.OSType == "windows", "Windows only supports 1.25 or later") defer setupTest(t)() client := testEnv.APIClient() @@ -103,6 +102,11 @@ func TestKillWithStopSignalAndRestartPolicies(t *testing.T) { }, } + var pollOpts []poll.SettingOp + if runtime.GOOS == "windows" { + pollOpts = append(pollOpts, poll.WithTimeout(StopContainerWindowsPollTimeout)) + } + for _, tc := range testCases { tc := tc t.Run(tc.doc, func(t *testing.T) { @@ -115,13 +119,12 @@ func TestKillWithStopSignalAndRestartPolicies(t *testing.T) { err := client.ContainerKill(ctx, id, "TERM") assert.NilError(t, err) - poll.WaitOn(t, container.IsInState(ctx, client, id, tc.status), poll.WithDelay(100*time.Millisecond)) + poll.WaitOn(t, container.IsInState(ctx, client, id, tc.status), pollOpts...) }) } } func TestKillStoppedContainer(t *testing.T) { - skip.If(t, testEnv.OSType == "windows", "Windows only supports 1.25 or later") defer setupTest(t)() ctx := context.Background() client := testEnv.APIClient() From b7d1e98ae73c11b449acc5065f8315f42d65a812 Mon Sep 17 00:00:00 2001 From: Albin Kerouanton Date: Tue, 30 May 2023 14:23:02 +0200 Subject: [PATCH 205/293] libnet/d/bridge: Allow IPv6 ICC from any IP address IPv6 ipt rules are exactly the same as IPv4 rules, although both protocol don't use the same networking model. This has bad consequences, for instance: 1. the current v6 rules disallow Neighbor Solication/Advertisement ; 2. multicast addresses can't be used ; 3. link-local addresses are blocked too. To solve this, this commit changes the following rules: ``` -A DOCKER-ISOLATION-STAGE-1 ! -s fdf1:a844:380c:b247::/64 -o br-21502e5b2c6c -j DROP -A DOCKER-ISOLATION-STAGE-1 ! -d fdf1:a844:380c:b247::/64 -i br-21502e5b2c6c -j DROP ``` into: ``` -A DOCKER-ISOLATION-STAGE-1 ! -s fdf1:a844:380c:b247::/64 ! -i br-21502e5b2c6c -o br-21502e5b2c6c -j DROP -A DOCKER-ISOLATION-STAGE-1 ! -d fdf1:a844:380c:b247::/64 -i br-21502e5b2c6c ! -o br-21502e5b2c6c -j DROP ``` These rules only limit the traffic ingressing/egressing the bridge, but not traffic between veth on the same bridge. Note that, the Kernel takes care of dropping invalid IPv6 packets, eg. loopback spoofing, thus these rules don't need to be more specific. Solve #45460. Signed-off-by: Albin Kerouanton (cherry picked from commit da9e44a620db39307f5548a1451be9c434c5b34d) Signed-off-by: Sebastiaan van Stijn --- libnetwork/drivers/bridge/setup_ip_tables.go | 23 +++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/libnetwork/drivers/bridge/setup_ip_tables.go b/libnetwork/drivers/bridge/setup_ip_tables.go index bd2822e39cab9..f53d0a93ae988 100644 --- a/libnetwork/drivers/bridge/setup_ip_tables.go +++ b/libnetwork/drivers/bridge/setup_ip_tables.go @@ -397,15 +397,21 @@ func removeIPChains(version iptables.IPVersion) { } func setupInternalNetworkRules(bridgeIface string, addr *net.IPNet, icc, insert bool) error { - var ( - inDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}} - outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}} - ) - - version := iptables.IPv4 - - if addr.IP.To4() == nil { + var version iptables.IPVersion + var inDropRule, outDropRule iptRule + + if addr.IP.To4() != nil { + version = iptables.IPv4 + inDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{ + "-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}} + outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{ + "-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}} + } else { version = iptables.IPv6 + inDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{ + "-i", bridgeIface, "!", "-o", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}} + outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{ + "!", "-i", bridgeIface, "-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}} } if err := programChainRule(version, inDropRule, "DROP INCOMING", insert); err != nil { @@ -414,6 +420,7 @@ func setupInternalNetworkRules(bridgeIface string, addr *net.IPNet, icc, insert if err := programChainRule(version, outDropRule, "DROP OUTGOING", insert); err != nil { return err } + // Set Inter Container Communication. return setIcc(version, bridgeIface, icc, insert) } From 25b709df48addea1b25ca92189078c204f214303 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 27 Jul 2023 16:20:49 +0200 Subject: [PATCH 206/293] windows: fix --register-service when executed from within binary directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Go 1.15.7 contained a security fix for CVE-2021-3115, which allowed arbitrary code to be executed at build time when using cgo on Windows. This issue was not limited to the go command itself, and could also affect binaries that use `os.Command`, `os.LookPath`, etc. From the related blogpost (https://blog.golang.org/path-security): > Are your own programs affected? > > If you use exec.LookPath or exec.Command in your own programs, you only need to > be concerned if you (or your users) run your program in a directory with untrusted > contents. If so, then a subprocess could be started using an executable from dot > instead of from a system directory. (Again, using an executable from dot happens > always on Windows and only with uncommon PATH settings on Unix.) > > If you are concerned, then we’ve published the more restricted variant of os/exec > as golang.org/x/sys/execabs. You can use it in your program by simply replacing At time of the go1.15 release, the Go team considered changing the behavior of `os.LookPath()` and `exec.LookPath()` to be a breaking change, and made the behavior "opt-in" by providing the `golang.org/x/sys/execabs` package as a replacement. However, for the go1.19 release, this changed, and the default behavior of `os.LookPath()` and `exec.LookPath()` was changed. From the release notes: https://go.dev/doc/go1.19#os-exec-path > Command and LookPath no longer allow results from a PATH search to be found > relative to the current directory. This removes a common source of security > problems but may also break existing programs that depend on using, say, > exec.Command("prog") to run a binary named prog (or, on Windows, prog.exe) > in the current directory. See the os/exec package documentation for information > about how best to update such programs. > > On Windows, Command and LookPath now respect the NoDefaultCurrentDirectoryInExePath > environment variable, making it possible to disable the default implicit search > of “.” in PATH lookups on Windows systems. A result of this change was that registering the daemon as a Windows service no longer worked when done from within the directory of the binary itself: C:\> cd "Program Files\Docker\Docker\resources" C:\Program Files\Docker\Docker\resources> dockerd --register-service exec: "dockerd": cannot run executable found relative to current directory Note that using an absolute path would work around the issue: C:\Program Files\Docker\Docker>resources\dockerd.exe --register-service This patch changes `registerService()` to use `os.Executable()`, instead of depending on `os.Args[0]` and `exec.LookPath()` for resolving the absolute path of the binary. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 3e8fda0a709bb7e706547d7b656e380eba965995) Signed-off-by: Sebastiaan van Stijn --- cmd/dockerd/service_windows.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/cmd/dockerd/service_windows.go b/cmd/dockerd/service_windows.go index 4510d79f88db8..cb7502d447a45 100644 --- a/cmd/dockerd/service_windows.go +++ b/cmd/dockerd/service_windows.go @@ -7,7 +7,6 @@ import ( "io" "log" "os" - "os/exec" "path/filepath" "time" @@ -145,16 +144,8 @@ func (h *etwHook) Fire(e *logrus.Entry) error { return windows.ReportEvent(h.log.Handle, etype, 0, eid, 0, count, 0, &ss[0], nil) } -func getServicePath() (string, error) { - p, err := exec.LookPath(os.Args[0]) - if err != nil { - return "", err - } - return filepath.Abs(p) -} - func registerService() error { - p, err := getServicePath() + p, err := os.Executable() if err != nil { return err } From 5f0df8c53429c03e9746bbbf1355d106f7f8e748 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 13 Aug 2023 22:08:44 +0200 Subject: [PATCH 207/293] vendor github.com/containerd/ttrpc v1.1.2 full diff: https://github.com/containerd/ttrpc/compare/v1.1.1...v1.1.2 Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 ++-- vendor/github.com/containerd/ttrpc/server.go | 2 +- vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vendor.mod b/vendor.mod index 93b49543cb836..8212b6aac36fb 100644 --- a/vendor.mod +++ b/vendor.mod @@ -121,7 +121,7 @@ require ( github.com/containerd/go-runc v1.1.0 // indirect github.com/containerd/nydus-snapshotter v0.3.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.13.0 // indirect - github.com/containerd/ttrpc v1.1.1 // indirect + github.com/containerd/ttrpc v1.1.2 // indirect github.com/containerd/typeurl v1.0.2 // indirect github.com/containernetworking/cni v1.1.1 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect diff --git a/vendor.sum b/vendor.sum index 7a265df2c3284..a27f32c1a2b78 100644 --- a/vendor.sum +++ b/vendor.sum @@ -419,8 +419,8 @@ github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0x github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/ttrpc v1.1.1 h1:NoRHS/z8UiHhpY1w0xcOqoJDGf2DHyzXrF0H4l5AE8c= -github.com/containerd/ttrpc v1.1.1/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/ttrpc v1.1.2 h1:4jH6OQDQqjfVD2b5TJS5TxmGuLGmp5WW7KtW2TWOP7c= +github.com/containerd/ttrpc v1.1.2/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= diff --git a/vendor/github.com/containerd/ttrpc/server.go b/vendor/github.com/containerd/ttrpc/server.go index e4c07b60fb8ce..5c62d169f3627 100644 --- a/vendor/github.com/containerd/ttrpc/server.go +++ b/vendor/github.com/containerd/ttrpc/server.go @@ -468,7 +468,7 @@ func (c *serverConn) run(sctx context.Context) { // branch. Basically, it means that we are no longer receiving // requests due to a terminal error. recvErr = nil // connection is now "closing" - if err == io.EOF || err == io.ErrUnexpectedEOF || errors.Is(err, syscall.ECONNRESET) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) { // The client went away and we should stop processing // requests, so that the client connection is closed return diff --git a/vendor/modules.txt b/vendor/modules.txt index 65ad85f86de6e..4baa79bd2ac05 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -323,7 +323,7 @@ github.com/containerd/nydus-snapshotter/pkg/errdefs ## explicit; go 1.16 github.com/containerd/stargz-snapshotter/estargz github.com/containerd/stargz-snapshotter/estargz/errorutil -# github.com/containerd/ttrpc v1.1.1 +# github.com/containerd/ttrpc v1.1.2 ## explicit; go 1.13 github.com/containerd/ttrpc # github.com/containerd/typeurl v1.0.2 From 7e7bc0f1bcb90c206ab04f958bc9b4c9f8349a34 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 29 Jul 2023 19:54:59 +0200 Subject: [PATCH 208/293] vendor: github.com/containerd/containerd v1.6.22 - full diff: https://github.com/containerd/containerd/compare/v1.6.21...v1.6.22 - release notes: https://github.com/containerd/containerd/releases/tag/v1.6.22 --- Notable Updates - RunC: Update runc binary to v1.1.8 - CRI: Fix `additionalGids`: it should fallback to `imageConfig.User` when `securityContext.RunAsUser`, `RunAsUsername` are empty - CRI: Write generated CNI config atomically - Fix concurrent writes for `UpdateContainerStats` - Make `checkContainerTimestamps` less strict on Windows - Port-Forward: Correctly handle known errors - Resolve `docker.NewResolver` race condition - SecComp: Always allow `name_to_handle_at` - Adding support to run hcsshim from local clone - Pinned image support - Runtime/V2/RunC: Handle early exits w/o big locks - CRITool: Move up to CRI-TOOLS v1.27.0 - Fix cpu architecture detection issue on emulated ARM platform - Task: Don't `close()` io before `cancel()` - Fix panic when remote differ returns empty result - Plugins: Notify readiness when registered plugins are ready - Unwrap io errors in server connection receive error handling Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 4d674897f3748a28347f2233237dbfae7c18cdc1) Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 +- .../github.com/containerd/containerd/.mailmap | 2 + .../github.com/containerd/containerd/Makefile | 16 +- .../containerd/containerd/Vagrantfile | 2 +- .../github.com/containerd/containerd/diff.go | 3 + .../containerd/containerd/log/context.go | 51 ++++++ .../containerd/pkg/atomicfile/file.go | 148 ++++++++++++++++ .../containerd/platforms/cpuinfo.go | 98 +---------- .../containerd/platforms/cpuinfo_linux.go | 161 ++++++++++++++++++ .../containerd/platforms/cpuinfo_other.go | 60 +++++++ .../containerd/platforms/database.go | 7 - .../containerd/containerd/plugin/context.go | 13 +- .../containerd/remotes/docker/resolver.go | 25 ++- .../containerd/runtime/v2/shim/util.go | 20 +-- .../github.com/containerd/containerd/task.go | 11 +- .../containerd/containerd/version/version.go | 2 +- vendor/modules.txt | 5 +- 18 files changed, 492 insertions(+), 138 deletions(-) create mode 100644 vendor/github.com/containerd/containerd/pkg/atomicfile/file.go create mode 100644 vendor/github.com/containerd/containerd/platforms/cpuinfo_linux.go create mode 100644 vendor/github.com/containerd/containerd/platforms/cpuinfo_other.go diff --git a/vendor.mod b/vendor.mod index 8212b6aac36fb..7cda78ef2bd90 100644 --- a/vendor.mod +++ b/vendor.mod @@ -25,7 +25,7 @@ require ( github.com/bsphere/le_go v0.0.0-20200109081728-fc06dab2caa8 github.com/cloudflare/cfssl v0.0.0-20180323000720-5d63dbd981b5 github.com/containerd/cgroups/v3 v3.0.2 - github.com/containerd/containerd v1.6.21 + github.com/containerd/containerd v1.6.22 github.com/containerd/continuity v0.3.0 github.com/containerd/fifo v1.1.0 github.com/containerd/typeurl/v2 v2.1.0 diff --git a/vendor.sum b/vendor.sum index a27f32c1a2b78..04af7872226f0 100644 --- a/vendor.sum +++ b/vendor.sum @@ -370,8 +370,8 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.6.21 h1:eSTAmnvDKRPWan+MpSSfNyrtleXd86ogK9X8fMWpe/Q= -github.com/containerd/containerd v1.6.21/go.mod h1:apei1/i5Ux2FzrK6+DM/suEsGuK/MeVOfy8tR2q7Wnw= +github.com/containerd/containerd v1.6.22 h1:rGTIBxPJusM0evF6wKgIzuD+tV70nmx9eEjzHVm1JzI= +github.com/containerd/containerd v1.6.22/go.mod h1:BQAJdahvGz8xboAvxKg9hsDYIovn79Ea318anowQ1/o= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= diff --git a/vendor/github.com/containerd/containerd/.mailmap b/vendor/github.com/containerd/containerd/.mailmap index 11dcdc48c0883..3988d4797a90e 100644 --- a/vendor/github.com/containerd/containerd/.mailmap +++ b/vendor/github.com/containerd/containerd/.mailmap @@ -145,3 +145,5 @@ Zhoulin Xie Zhoulin Xie <42261994+JoeWrightss@users.noreply.github.com> zounengren 张潇 +Kazuyoshi Kato +Andrey Epifanov diff --git a/vendor/github.com/containerd/containerd/Makefile b/vendor/github.com/containerd/containerd/Makefile index 7441eeac66e12..f1b28ceb9c9f5 100644 --- a/vendor/github.com/containerd/containerd/Makefile +++ b/vendor/github.com/containerd/containerd/Makefile @@ -332,22 +332,26 @@ install-cri-deps: $(BINARIES) @$(INSTALL) $(BINARIES) $(CRIDIR)/bin endif +$(CRIDIR)/cri-containerd.DEPRECATED.txt: + @mkdir -p $(CRIDIR) + @$(INSTALL) -m 644 releases/cri-containerd.DEPRECATED.txt $@ + ifeq ($(GOOS),windows) -releases/$(CRIRELEASE).tar.gz: install-cri-deps +releases/$(CRIRELEASE).tar.gz: install-cri-deps $(CRIDIR)/cri-containerd.DEPRECATED.txt @echo "$(WHALE) $@" @cd $(CRIDIR) && tar -czf ../../releases/$(CRIRELEASE).tar.gz * -releases/$(CRICNIRELEASE).tar.gz: install-cri-deps +releases/$(CRICNIRELEASE).tar.gz: install-cri-deps $(CRIDIR)/cri-containerd.DEPRECATED.txt @echo "$(WHALE) $@" @cd $(CRIDIR) && tar -czf ../../releases/$(CRICNIRELEASE).tar.gz * else -releases/$(CRIRELEASE).tar.gz: install-cri-deps +releases/$(CRIRELEASE).tar.gz: install-cri-deps $(CRIDIR)/cri-containerd.DEPRECATED.txt @echo "$(WHALE) $@" - @tar -czf releases/$(CRIRELEASE).tar.gz -C $(CRIDIR) etc/crictl.yaml etc/systemd usr opt/containerd + @tar -czf releases/$(CRIRELEASE).tar.gz -C $(CRIDIR) cri-containerd.DEPRECATED.txt etc/crictl.yaml etc/systemd usr opt/containerd -releases/$(CRICNIRELEASE).tar.gz: install-cri-deps +releases/$(CRICNIRELEASE).tar.gz: install-cri-deps $(CRIDIR)/cri-containerd.DEPRECATED.txt @echo "$(WHALE) $@" - @tar -czf releases/$(CRICNIRELEASE).tar.gz -C $(CRIDIR) etc usr opt + @tar -czf releases/$(CRICNIRELEASE).tar.gz -C $(CRIDIR) cri-containerd.DEPRECATED.txt etc usr opt endif cri-release: releases/$(CRIRELEASE).tar.gz diff --git a/vendor/github.com/containerd/containerd/Vagrantfile b/vendor/github.com/containerd/containerd/Vagrantfile index f706788eccdda..95c3a359b41a5 100644 --- a/vendor/github.com/containerd/containerd/Vagrantfile +++ b/vendor/github.com/containerd/containerd/Vagrantfile @@ -93,7 +93,7 @@ EOF config.vm.provision "install-golang", type: "shell", run: "once" do |sh| sh.upload_path = "/tmp/vagrant-install-golang" sh.env = { - 'GO_VERSION': ENV['GO_VERSION'] || "1.19.9", + 'GO_VERSION': ENV['GO_VERSION'] || "1.19.11", } sh.inline = <<~SHELL #!/usr/bin/env bash diff --git a/vendor/github.com/containerd/containerd/diff.go b/vendor/github.com/containerd/containerd/diff.go index 445df019220c8..28012b1f02bc5 100644 --- a/vendor/github.com/containerd/containerd/diff.go +++ b/vendor/github.com/containerd/containerd/diff.go @@ -86,6 +86,9 @@ func (r *diffRemote) Compare(ctx context.Context, a, b []mount.Mount, opts ...di } func toDescriptor(d *types.Descriptor) ocispec.Descriptor { + if d == nil { + return ocispec.Descriptor{} + } return ocispec.Descriptor{ MediaType: d.MediaType, Digest: d.Digest, diff --git a/vendor/github.com/containerd/containerd/log/context.go b/vendor/github.com/containerd/containerd/log/context.go index 0db9562b82bba..b63c602f424a5 100644 --- a/vendor/github.com/containerd/containerd/log/context.go +++ b/vendor/github.com/containerd/containerd/log/context.go @@ -18,6 +18,7 @@ package log import ( "context" + "fmt" "github.com/sirupsen/logrus" ) @@ -35,6 +36,12 @@ var ( type ( loggerKey struct{} + + // Fields type to pass to `WithFields`, alias from `logrus`. + Fields = logrus.Fields + + // Level is a logging level + Level = logrus.Level ) const ( @@ -47,8 +54,52 @@ const ( // JSONFormat represents the JSON logging format JSONFormat = "json" + + // TraceLevel level. + TraceLevel = logrus.TraceLevel + + // DebugLevel level. + DebugLevel = logrus.DebugLevel + + // InfoLevel level. + InfoLevel = logrus.InfoLevel ) +// SetLevel sets log level globally. +func SetLevel(level string) error { + lvl, err := logrus.ParseLevel(level) + if err != nil { + return err + } + + logrus.SetLevel(lvl) + return nil +} + +// GetLevel returns the current log level. +func GetLevel() Level { + return logrus.GetLevel() +} + +// SetFormat sets log output format +func SetFormat(format string) error { + switch format { + case TextFormat: + logrus.SetFormatter(&logrus.TextFormatter{ + TimestampFormat: RFC3339NanoFixed, + FullTimestamp: true, + }) + case JSONFormat: + logrus.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: RFC3339NanoFixed, + }) + default: + return fmt.Errorf("unknown log format: %s", format) + } + + return nil +} + // WithLogger returns a new context with the provided logger. Use in // combination with logger.WithField(s) for great effect. func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context { diff --git a/vendor/github.com/containerd/containerd/pkg/atomicfile/file.go b/vendor/github.com/containerd/containerd/pkg/atomicfile/file.go new file mode 100644 index 0000000000000..7b870f7a787d7 --- /dev/null +++ b/vendor/github.com/containerd/containerd/pkg/atomicfile/file.go @@ -0,0 +1,148 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* +Package atomicfile provides a mechanism (on Unix-like platforms) to present a consistent view of a file to separate +processes even while the file is being written. This is accomplished by writing a temporary file, syncing to disk, and +renaming over the destination file name. + +Partial/inconsistent reads can occur due to: + 1. A process attempting to read the file while it is being written to (both in the case of a new file with a + short/incomplete write or in the case of an existing, updated file where new bytes may be written at the beginning + but old bytes may still be present after). + 2. Concurrent goroutines leading to multiple active writers of the same file. + +The above mechanism explicitly protects against (1) as all writes are to a file with a temporary name. + +There is no explicit protection against multiple, concurrent goroutines attempting to write the same file. However, +atomically writing the file should mean only one writer will "win" and a consistent file will be visible. + +Note: atomicfile is partially implemented for Windows. The Windows codepath performs the same operations, however +Windows does not guarantee that a rename operation is atomic; a crash in the middle may leave the destination file +truncated rather than with the expected content. +*/ +package atomicfile + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" +) + +// File is an io.ReadWriteCloser that can also be Canceled if a change needs to be abandoned. +type File interface { + io.ReadWriteCloser + // Cancel abandons a change to a file. This can be called if a write fails or another error occurs. + Cancel() error +} + +// ErrClosed is returned if Read or Write are called on a closed File. +var ErrClosed = errors.New("file is closed") + +// New returns a new atomic file. On Unix-like platforms, the writer (an io.ReadWriteCloser) is backed by a temporary +// file placed into the same directory as the destination file (using filepath.Dir to split the directory from the +// name). On a call to Close the temporary file is synced to disk and renamed to its final name, hiding any previous +// file by the same name. +// +// Note: Take care to call Close and handle any errors that are returned. Errors returned from Close may indicate that +// the file was not written with its final name. +func New(name string, mode os.FileMode) (File, error) { + return newFile(name, mode) +} + +type atomicFile struct { + name string + f *os.File + closed bool + closedMu sync.RWMutex +} + +func newFile(name string, mode os.FileMode) (File, error) { + dir := filepath.Dir(name) + f, err := os.CreateTemp(dir, "") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + if err := f.Chmod(mode); err != nil { + return nil, fmt.Errorf("failed to change temp file permissions: %w", err) + } + return &atomicFile{name: name, f: f}, nil +} + +func (a *atomicFile) Close() (err error) { + a.closedMu.Lock() + defer a.closedMu.Unlock() + + if a.closed { + return nil + } + a.closed = true + + defer func() { + if err != nil { + _ = os.Remove(a.f.Name()) // ignore errors + } + }() + // The order of operations here is: + // 1. sync + // 2. close + // 3. rename + // While the ordering of 2 and 3 is not important on Unix-like operating systems, Windows cannot rename an open + // file. By closing first, we allow the rename operation to succeed. + if err = a.f.Sync(); err != nil { + return fmt.Errorf("failed to sync temp file %q: %w", a.f.Name(), err) + } + if err = a.f.Close(); err != nil { + return fmt.Errorf("failed to close temp file %q: %w", a.f.Name(), err) + } + if err = os.Rename(a.f.Name(), a.name); err != nil { + return fmt.Errorf("failed to rename %q to %q: %w", a.f.Name(), a.name, err) + } + return nil +} + +func (a *atomicFile) Cancel() error { + a.closedMu.Lock() + defer a.closedMu.Unlock() + + if a.closed { + return nil + } + a.closed = true + _ = a.f.Close() // ignore error + return os.Remove(a.f.Name()) +} + +func (a *atomicFile) Read(p []byte) (n int, err error) { + a.closedMu.RLock() + defer a.closedMu.RUnlock() + if a.closed { + return 0, ErrClosed + } + return a.f.Read(p) +} + +func (a *atomicFile) Write(p []byte) (n int, err error) { + a.closedMu.RLock() + defer a.closedMu.RUnlock() + if a.closed { + return 0, ErrClosed + } + return a.f.Write(p) +} diff --git a/vendor/github.com/containerd/containerd/platforms/cpuinfo.go b/vendor/github.com/containerd/containerd/platforms/cpuinfo.go index 046e0356d19db..8c600fc96b1c7 100644 --- a/vendor/github.com/containerd/containerd/platforms/cpuinfo.go +++ b/vendor/github.com/containerd/containerd/platforms/cpuinfo.go @@ -17,14 +17,9 @@ package platforms import ( - "bufio" - "fmt" - "os" "runtime" - "strings" "sync" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" ) @@ -37,95 +32,12 @@ var cpuVariantOnce sync.Once func cpuVariant() string { cpuVariantOnce.Do(func() { if isArmArch(runtime.GOARCH) { - cpuVariantValue = getCPUVariant() + var err error + cpuVariantValue, err = getCPUVariant() + if err != nil { + log.L.Errorf("Error getCPUVariant for OS %s: %v", runtime.GOOS, err) + } } }) return cpuVariantValue } - -// For Linux, the kernel has already detected the ABI, ISA and Features. -// So we don't need to access the ARM registers to detect platform information -// by ourselves. We can just parse these information from /proc/cpuinfo -func getCPUInfo(pattern string) (info string, err error) { - if !isLinuxOS(runtime.GOOS) { - return "", fmt.Errorf("getCPUInfo for OS %s: %w", runtime.GOOS, errdefs.ErrNotImplemented) - } - - cpuinfo, err := os.Open("/proc/cpuinfo") - if err != nil { - return "", err - } - defer cpuinfo.Close() - - // Start to Parse the Cpuinfo line by line. For SMP SoC, we parse - // the first core is enough. - scanner := bufio.NewScanner(cpuinfo) - for scanner.Scan() { - newline := scanner.Text() - list := strings.Split(newline, ":") - - if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) { - return strings.TrimSpace(list[1]), nil - } - } - - // Check whether the scanner encountered errors - err = scanner.Err() - if err != nil { - return "", err - } - - return "", fmt.Errorf("getCPUInfo for pattern: %s: %w", pattern, errdefs.ErrNotFound) -} - -func getCPUVariant() string { - if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { - // Windows/Darwin only supports v7 for ARM32 and v8 for ARM64 and so we can use - // runtime.GOARCH to determine the variants - var variant string - switch runtime.GOARCH { - case "arm64": - variant = "v8" - case "arm": - variant = "v7" - default: - variant = "unknown" - } - - return variant - } - - variant, err := getCPUInfo("Cpu architecture") - if err != nil { - log.L.WithError(err).Error("failure getting variant") - return "" - } - - // handle edge case for Raspberry Pi ARMv6 devices (which due to a kernel quirk, report "CPU architecture: 7") - // https://www.raspberrypi.org/forums/viewtopic.php?t=12614 - if runtime.GOARCH == "arm" && variant == "7" { - model, err := getCPUInfo("model name") - if err == nil && strings.HasPrefix(strings.ToLower(model), "armv6-compatible") { - variant = "6" - } - } - - switch strings.ToLower(variant) { - case "8", "aarch64": - variant = "v8" - case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)": - variant = "v7" - case "6", "6tej": - variant = "v6" - case "5", "5t", "5te", "5tej": - variant = "v5" - case "4", "4t": - variant = "v4" - case "3": - variant = "v3" - default: - variant = "unknown" - } - - return variant -} diff --git a/vendor/github.com/containerd/containerd/platforms/cpuinfo_linux.go b/vendor/github.com/containerd/containerd/platforms/cpuinfo_linux.go new file mode 100644 index 0000000000000..722d86c3578cc --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/cpuinfo_linux.go @@ -0,0 +1,161 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import ( + "bufio" + "bytes" + "fmt" + "os" + "runtime" + "strings" + + "github.com/containerd/containerd/errdefs" + "golang.org/x/sys/unix" +) + +// getMachineArch retrieves the machine architecture through system call +func getMachineArch() (string, error) { + var uname unix.Utsname + err := unix.Uname(&uname) + if err != nil { + return "", err + } + + arch := string(uname.Machine[:bytes.IndexByte(uname.Machine[:], 0)]) + + return arch, nil +} + +// For Linux, the kernel has already detected the ABI, ISA and Features. +// So we don't need to access the ARM registers to detect platform information +// by ourselves. We can just parse these information from /proc/cpuinfo +func getCPUInfo(pattern string) (info string, err error) { + + cpuinfo, err := os.Open("/proc/cpuinfo") + if err != nil { + return "", err + } + defer cpuinfo.Close() + + // Start to Parse the Cpuinfo line by line. For SMP SoC, we parse + // the first core is enough. + scanner := bufio.NewScanner(cpuinfo) + for scanner.Scan() { + newline := scanner.Text() + list := strings.Split(newline, ":") + + if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) { + return strings.TrimSpace(list[1]), nil + } + } + + // Check whether the scanner encountered errors + err = scanner.Err() + if err != nil { + return "", err + } + + return "", fmt.Errorf("getCPUInfo for pattern %s: %w", pattern, errdefs.ErrNotFound) +} + +// getCPUVariantFromArch get CPU variant from arch through a system call +func getCPUVariantFromArch(arch string) (string, error) { + + var variant string + + arch = strings.ToLower(arch) + + if arch == "aarch64" { + variant = "8" + } else if arch[0:4] == "armv" && len(arch) >= 5 { + //Valid arch format is in form of armvXx + switch arch[3:5] { + case "v8": + variant = "8" + case "v7": + variant = "7" + case "v6": + variant = "6" + case "v5": + variant = "5" + case "v4": + variant = "4" + case "v3": + variant = "3" + default: + variant = "unknown" + } + } else { + return "", fmt.Errorf("getCPUVariantFromArch invalid arch: %s, %w", arch, errdefs.ErrInvalidArgument) + } + return variant, nil +} + +// getCPUVariant returns cpu variant for ARM +// We first try reading "Cpu architecture" field from /proc/cpuinfo +// If we can't find it, then fall back using a system call +// This is to cover running ARM in emulated environment on x86 host as this field in /proc/cpuinfo +// was not present. +func getCPUVariant() (string, error) { + + variant, err := getCPUInfo("Cpu architecture") + if err != nil { + if errdefs.IsNotFound(err) { + //Let's try getting CPU variant from machine architecture + arch, err := getMachineArch() + if err != nil { + return "", fmt.Errorf("failure getting machine architecture: %v", err) + } + + variant, err = getCPUVariantFromArch(arch) + if err != nil { + return "", fmt.Errorf("failure getting CPU variant from machine architecture: %v", err) + } + } else { + return "", fmt.Errorf("failure getting CPU variant: %v", err) + } + } + + // handle edge case for Raspberry Pi ARMv6 devices (which due to a kernel quirk, report "CPU architecture: 7") + // https://www.raspberrypi.org/forums/viewtopic.php?t=12614 + if runtime.GOARCH == "arm" && variant == "7" { + model, err := getCPUInfo("model name") + if err == nil && strings.HasPrefix(strings.ToLower(model), "armv6-compatible") { + variant = "6" + } + } + + switch strings.ToLower(variant) { + case "8", "aarch64": + variant = "v8" + case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)": + variant = "v7" + case "6", "6tej": + variant = "v6" + case "5", "5t", "5te", "5tej": + variant = "v5" + case "4", "4t": + variant = "v4" + case "3": + variant = "v3" + default: + variant = "unknown" + } + + return variant, nil +} diff --git a/vendor/github.com/containerd/containerd/platforms/cpuinfo_other.go b/vendor/github.com/containerd/containerd/platforms/cpuinfo_other.go new file mode 100644 index 0000000000000..51fb62ea7195b --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/cpuinfo_other.go @@ -0,0 +1,60 @@ +//go:build !linux +// +build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import ( + "fmt" + "runtime" + + "github.com/containerd/containerd/errdefs" +) + +func getCPUVariant() (string, error) { + + var variant string + + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + // Windows/Darwin only supports v7 for ARM32 and v8 for ARM64 and so we can use + // runtime.GOARCH to determine the variants + switch runtime.GOARCH { + case "arm64": + variant = "v8" + case "arm": + variant = "v7" + default: + variant = "unknown" + } + } else if runtime.GOOS == "freebsd" { + // FreeBSD supports ARMv6 and ARMv7 as well as ARMv4 and ARMv5 (though deprecated) + // detecting those variants is currently unimplemented + switch runtime.GOARCH { + case "arm64": + variant = "v8" + default: + variant = "unknown" + } + + } else { + return "", fmt.Errorf("getCPUVariant for OS %s: %v", runtime.GOOS, errdefs.ErrNotImplemented) + + } + + return variant, nil +} diff --git a/vendor/github.com/containerd/containerd/platforms/database.go b/vendor/github.com/containerd/containerd/platforms/database.go index dbe9957ca9dbd..2e26fd3b4faed 100644 --- a/vendor/github.com/containerd/containerd/platforms/database.go +++ b/vendor/github.com/containerd/containerd/platforms/database.go @@ -21,13 +21,6 @@ import ( "strings" ) -// isLinuxOS returns true if the operating system is Linux. -// -// The OS value should be normalized before calling this function. -func isLinuxOS(os string) bool { - return os == "linux" -} - // These function are generated from https://golang.org/src/go/build/syslist.go. // // We use switch statements because they are slightly faster than map lookups diff --git a/vendor/github.com/containerd/containerd/plugin/context.go b/vendor/github.com/containerd/containerd/plugin/context.go index dcb533c8a7454..cf91678988126 100644 --- a/vendor/github.com/containerd/containerd/plugin/context.go +++ b/vendor/github.com/containerd/containerd/plugin/context.go @@ -28,12 +28,13 @@ import ( // InitContext is used for plugin initialization type InitContext struct { - Context context.Context - Root string - State string - Config interface{} - Address string - TTRPCAddress string + Context context.Context + Root string + State string + Config interface{} + Address string + TTRPCAddress string + RegisterReadiness func() func() // deprecated: will be removed in 2.0, use plugin.EventType Events *exchange.Exchange diff --git a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go index 709fa028de278..13f500e4d2274 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go @@ -95,25 +95,30 @@ type ResolverOptions struct { Tracker StatusTracker // Authorizer is used to authorize registry requests - // Deprecated: use Hosts + // + // Deprecated: use Hosts. Authorizer Authorizer // Credentials provides username and secret given a host. // If username is empty but a secret is given, that secret // is interpreted as a long lived token. - // Deprecated: use Hosts + // + // Deprecated: use Hosts. Credentials func(string) (string, string, error) // Host provides the hostname given a namespace. - // Deprecated: use Hosts + // + // Deprecated: use Hosts. Host func(string) (string, error) // PlainHTTP specifies to use plain http and not https - // Deprecated: use Hosts + // + // Deprecated: use Hosts. PlainHTTP bool // Client is the http client to used when making registry requests - // Deprecated: use Hosts + // + // Deprecated: use Hosts. Client *http.Client } @@ -140,6 +145,9 @@ func NewResolver(options ResolverOptions) remotes.Resolver { if options.Headers == nil { options.Headers = make(http.Header) + } else { + // make a copy of the headers to avoid race due to concurrent map write + options.Headers = options.Headers.Clone() } if _, ok := options.Headers["User-Agent"]; !ok { options.Headers.Set("User-Agent", "containerd/"+version.Version) @@ -529,9 +537,10 @@ func (r *request) do(ctx context.Context) (*http.Response, error) { if err != nil { return nil, err } - req.Header = http.Header{} // headers need to be copied to avoid concurrent map access - for k, v := range r.header { - req.Header[k] = v + if r.header == nil { + req.Header = http.Header{} + } else { + req.Header = r.header.Clone() // headers need to be copied to avoid concurrent map access } if r.body != nil { body, err := r.body() diff --git a/vendor/github.com/containerd/containerd/runtime/v2/shim/util.go b/vendor/github.com/containerd/containerd/runtime/v2/shim/util.go index 1a0d41f23172e..94c0f5397855a 100644 --- a/vendor/github.com/containerd/containerd/runtime/v2/shim/util.go +++ b/vendor/github.com/containerd/containerd/runtime/v2/shim/util.go @@ -27,11 +27,13 @@ import ( "strings" "time" - "github.com/containerd/containerd/namespaces" "github.com/containerd/ttrpc" "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" exec "golang.org/x/sys/execabs" + + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/pkg/atomicfile" ) type CommandConfig struct { @@ -118,17 +120,16 @@ func WritePidFile(path string, pid int) error { if err != nil { return err } - tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path))) - f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666) + f, err := atomicfile.New(path, 0o666) if err != nil { return err } _, err = fmt.Fprintf(f, "%d", pid) - f.Close() if err != nil { + f.Cancel() return err } - return os.Rename(tempPath, path) + return f.Close() } // WriteAddress writes a address file atomically @@ -137,17 +138,16 @@ func WriteAddress(path, address string) error { if err != nil { return err } - tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path))) - f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666) + f, err := atomicfile.New(path, 0o666) if err != nil { return err } - _, err = f.WriteString(address) - f.Close() + _, err = f.Write([]byte(address)) if err != nil { + f.Cancel() return err } - return os.Rename(tempPath, path) + return f.Close() } // ErrNoAddress is returned when the address file has no content diff --git a/vendor/github.com/containerd/containerd/task.go b/vendor/github.com/containerd/containerd/task.go index 9be1394cf4643..ef8cd44942cfb 100644 --- a/vendor/github.com/containerd/containerd/task.go +++ b/vendor/github.com/containerd/containerd/task.go @@ -325,7 +325,16 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStat return nil, fmt.Errorf("task must be stopped before deletion: %s: %w", status.Status, errdefs.ErrFailedPrecondition) } if t.io != nil { - t.io.Close() + // io.Wait locks for restored tasks on Windows unless we call + // io.Close first (https://github.com/containerd/containerd/issues/5621) + // in other cases, preserve the contract and let IO finish before closing + if t.client.runtime == fmt.Sprintf("%s.%s", plugin.RuntimePlugin, "windows") { + t.io.Close() + } + // io.Cancel is used to cancel the io goroutine while it is in + // fifo-opening state. It does not stop the pipes since these + // should be closed on the shim's side, otherwise we might lose + // data from the container! t.io.Cancel() t.io.Wait() } diff --git a/vendor/github.com/containerd/containerd/version/version.go b/vendor/github.com/containerd/containerd/version/version.go index 2fee285ac1bc9..de124ef60aa5e 100644 --- a/vendor/github.com/containerd/containerd/version/version.go +++ b/vendor/github.com/containerd/containerd/version/version.go @@ -23,7 +23,7 @@ var ( Package = "github.com/containerd/containerd" // Version holds the complete version number. Filled in at linking time. - Version = "1.6.21+unknown" + Version = "1.6.22+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. diff --git a/vendor/modules.txt b/vendor/modules.txt index 4baa79bd2ac05..c3df1de562494 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -216,8 +216,8 @@ github.com/containerd/cgroups/v3/cgroup2/stats # github.com/containerd/console v1.0.3 ## explicit; go 1.13 github.com/containerd/console -# github.com/containerd/containerd v1.6.21 -## explicit; go 1.17 +# github.com/containerd/containerd v1.6.22 +## explicit; go 1.18 github.com/containerd/containerd github.com/containerd/containerd/api/events github.com/containerd/containerd/api/services/containers/v1 @@ -266,6 +266,7 @@ github.com/containerd/containerd/mount github.com/containerd/containerd/namespaces github.com/containerd/containerd/oci github.com/containerd/containerd/pkg/apparmor +github.com/containerd/containerd/pkg/atomicfile github.com/containerd/containerd/pkg/cap github.com/containerd/containerd/pkg/dialer github.com/containerd/containerd/pkg/kmutex From 72947f5022017ad11837ffa815023c2f2c70350c Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Tue, 11 Jul 2023 07:36:03 -0600 Subject: [PATCH 209/293] hack: use Git-free ROOTDIR convention Signed-off-by: Bjorn Neergaard (cherry picked from commit 5563b09ac242fb07e031e935c6f28f15baa41e62) Signed-off-by: Sebastiaan van Stijn --- hack/generate-authors.sh | 2 +- hack/validate/no-module | 2 +- hack/with-go-mod.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/generate-authors.sh b/hack/generate-authors.sh index 5133ed5b023c3..da30edb5fbded 100755 --- a/hack/generate-authors.sh +++ b/hack/generate-authors.sh @@ -3,7 +3,7 @@ set -e SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" +ROOTDIR="$(cd "${SCRIPTDIR}/.." && pwd)" set -x diff --git a/hack/validate/no-module b/hack/validate/no-module index 67a9c559add54..917cc3e756fbd 100755 --- a/hack/validate/no-module +++ b/hack/validate/no-module @@ -3,7 +3,7 @@ # Check that no one is trying to commit a go.mod. SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" +ROOTDIR="$(cd "${SCRIPTDIR}/../.." && pwd)" if test -e "${ROOTDIR}/go.mod"; then { diff --git a/hack/with-go-mod.sh b/hack/with-go-mod.sh index e4210f73c479a..868d204998b3c 100755 --- a/hack/with-go-mod.sh +++ b/hack/with-go-mod.sh @@ -9,7 +9,7 @@ set -e SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" +ROOTDIR="$(cd "${SCRIPTDIR}/.." && pwd)" if test -e "${ROOTDIR}/go.mod"; then { From 314b84b023acd11eca9b4a5c075cba1b77a6bd8a Mon Sep 17 00:00:00 2001 From: Kevin Alvarez Date: Tue, 11 Apr 2023 02:31:23 +0200 Subject: [PATCH 210/293] hack: enable Go modules when building dockerd and docker-proxy This is a workaround to have buildinfo with deps embedded in the binary. We need to create a go.mod file before building with -modfile=vendor.mod, otherwise it fails with: "-modfile cannot be used to set the module root directory." Signed-off-by: CrazyMax (cherry picked from commit 7665feeb528d02021b943aed3655eff5eca96598) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- hack/make/.binary | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4cbee5021570..0b32c3c901a17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -599,7 +599,7 @@ RUN < "go.mod" + trap 'rm -f go.mod' EXIT + fi + echo "Building $([ "$DOCKER_STATIC" = "1" ] && echo "static" || echo "dynamic") $DEST/$BINARY_FULLNAME ($PLATFORM_NAME)..." if [ -n "$DOCKER_DEBUG" ]; then set -x fi - go build -o "$DEST/$BINARY_FULLNAME" "${BUILDFLAGS[@]}" -ldflags "$LDFLAGS $LDFLAGS_STATIC $DOCKER_LDFLAGS" ${GO_PACKAGE} + GO111MODULE=on go build -mod=vendor -modfile=vendor.mod -o "$DEST/$BINARY_FULLNAME" "${BUILDFLAGS[@]}" -ldflags "$LDFLAGS $LDFLAGS_STATIC $DOCKER_LDFLAGS" ${GO_PACKAGE} ) echo "Created binary: $DEST/$BINARY_FULLNAME" From e67f9dadc6cf5c7b4410e268d2a550cb18249f4a Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Tue, 11 Jul 2023 07:36:17 -0600 Subject: [PATCH 211/293] hack/make/.binary: use with-go-mod.sh Signed-off-by: Bjorn Neergaard (cherry picked from commit a972dbd682ec41e64a8ed95decf901ee040f003c) Signed-off-by: Sebastiaan van Stijn --- hack/make/.binary | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/hack/make/.binary b/hack/make/.binary index 90d16f11a250c..b73051f2a9531 100644 --- a/hack/make/.binary +++ b/hack/make/.binary @@ -74,19 +74,11 @@ source "${MAKEDIR}/.go-autogen" fi fi - # This is a workaround to have buildinfo with deps embedded in the binary. We - # need to create a go.mod file before building with -modfile=vendor.mod, - # otherwise it fails with: "-modfile cannot be used to set the module root directory." - if [ ! -f "go.mod" ]; then - printf '%s\n\n%s' 'module github.com/docker/docker' 'go 1.19' > "go.mod" - trap 'rm -f go.mod' EXIT - fi - echo "Building $([ "$DOCKER_STATIC" = "1" ] && echo "static" || echo "dynamic") $DEST/$BINARY_FULLNAME ($PLATFORM_NAME)..." if [ -n "$DOCKER_DEBUG" ]; then set -x fi - GO111MODULE=on go build -mod=vendor -modfile=vendor.mod -o "$DEST/$BINARY_FULLNAME" "${BUILDFLAGS[@]}" -ldflags "$LDFLAGS $LDFLAGS_STATIC $DOCKER_LDFLAGS" ${GO_PACKAGE} + ./hack/with-go-mod.sh go build -mod=vendor -modfile=vendor.mod -o "$DEST/$BINARY_FULLNAME" "${BUILDFLAGS[@]}" -ldflags "$LDFLAGS $LDFLAGS_STATIC $DOCKER_LDFLAGS" "$GO_PACKAGE" ) echo "Created binary: $DEST/$BINARY_FULLNAME" From bf2b8a05a0b7bd07eb35ab73d4e6af50651a8625 Mon Sep 17 00:00:00 2001 From: Luboslav Pivarc Date: Wed, 10 May 2023 10:09:21 +0200 Subject: [PATCH 212/293] Do not drop effective&permitted set Currently moby drops ep sets before the entrypoint is executed. This does mean that with combination of no-new-privileges the file capabilities stops working with non-root containers. This is undesired as the usability of such containers is harmed comparing to running root containers. This commit therefore sets the effective/permitted set in order to allow use of file capabilities or libcap(3)/prctl(2) respectively with combination of no-new-privileges and without respectively. For no-new-privileges the container will be able to obtain capabilities that are requested. Signed-off-by: Luboslav Pivarc Signed-off-by: Bjorn Neergaard (cherry picked from commit 3aef732e61ec8ae0ea0bd8ad31116194e0fc21a6) Signed-off-by: Sebastiaan van Stijn --- oci/oci.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/oci/oci.go b/oci/oci.go index 864ccf5b60c41..45ed7979ee827 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -23,19 +23,10 @@ func SetCapabilities(s *specs.Spec, caplist []string) error { if s.Process == nil { s.Process = &specs.Process{} } - // setUser has already been executed here - if s.Process.User.UID == 0 { - s.Process.Capabilities = &specs.LinuxCapabilities{ - Effective: caplist, - Bounding: caplist, - Permitted: caplist, - } - } else { - // Do not set Effective and Permitted capabilities for non-root users, - // to match what execve does. - s.Process.Capabilities = &specs.LinuxCapabilities{ - Bounding: caplist, - } + s.Process.Capabilities = &specs.LinuxCapabilities{ + Effective: caplist, + Bounding: caplist, + Permitted: caplist, } return nil } From cc39fb9f6b6863998f061f303f0a10377336cc6b Mon Sep 17 00:00:00 2001 From: Luboslav Pivarc Date: Mon, 17 Jul 2023 11:48:02 +0200 Subject: [PATCH 213/293] Integration test for capabilities Verify non-root containers are able to use file capabilities. Signed-off-by: Luboslav Pivarc Co-authored-by: Cory Snider Signed-off-by: Cory Snider (cherry picked from commit 42fa7a1951f192a0038b904f529e86beeb056093) Signed-off-by: Sebastiaan van Stijn --- .../capabilities/capabilities_linux_test.go | 108 ++++++++++++++++++ integration/capabilities/main_linux_test.go | 33 ++++++ integration/internal/container/ops.go | 18 +++ 3 files changed, 159 insertions(+) create mode 100644 integration/capabilities/capabilities_linux_test.go create mode 100644 integration/capabilities/main_linux_test.go diff --git a/integration/capabilities/capabilities_linux_test.go b/integration/capabilities/capabilities_linux_test.go new file mode 100644 index 0000000000000..272f3dcdb7c5c --- /dev/null +++ b/integration/capabilities/capabilities_linux_test.go @@ -0,0 +1,108 @@ +package capabilities + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/docker/testutil/fakecontext" + + "gotest.tools/v3/assert" + "gotest.tools/v3/poll" +) + +func TestNoNewPrivileges(t *testing.T) { + defer setupTest(t)() + + withFileCapability := ` + FROM debian:bullseye-slim + RUN apt-get update && apt-get install -y libcap2-bin --no-install-recommends + RUN setcap CAP_DAC_OVERRIDE=+eip /bin/cat + RUN echo "hello" > /txt && chown 0:0 /txt && chmod 700 /txt + RUN useradd -u 1500 test + ` + imageTag := "captest" + + source := fakecontext.New(t, "", fakecontext.WithDockerfile(withFileCapability)) + defer source.Close() + + client := testEnv.APIClient() + + // Build image + ctx := context.TODO() + resp, err := client.ImageBuild(ctx, + source.AsTarReader(t), + types.ImageBuildOptions{ + Tags: []string{imageTag}, + }) + assert.NilError(t, err) + _, err = io.Copy(io.Discard, resp.Body) + assert.NilError(t, err) + resp.Body.Close() + + testCases := []struct { + doc string + opts []func(*container.TestContainerConfig) + stdOut, stdErr string + }{ + { + doc: "CapabilityRequested=true", + opts: []func(*container.TestContainerConfig){ + container.WithUser("test"), + container.WithCapability("CAP_DAC_OVERRIDE"), + }, + stdOut: "hello", + }, + { + doc: "CapabilityRequested=false", + opts: []func(*container.TestContainerConfig){ + container.WithUser("test"), + container.WithDropCapability("CAP_DAC_OVERRIDE"), + }, + stdErr: "exec /bin/cat: operation not permitted", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.doc, func(t *testing.T) { + // Run the container with the image + opts := append(tc.opts, + container.WithImage(imageTag), + container.WithCmd("/bin/cat", "/txt"), + container.WithSecurityOpt("no-new-privileges=true"), + ) + cid := container.Run(ctx, t, client, opts...) + poll.WaitOn(t, container.IsInState(ctx, client, cid, "exited"), poll.WithDelay(100*time.Millisecond)) + + // Assert on outputs + logReader, err := client.ContainerLogs(ctx, cid, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + assert.NilError(t, err) + defer logReader.Close() + + var actualStdout, actualStderr bytes.Buffer + _, err = stdcopy.StdCopy(&actualStdout, &actualStderr, logReader) + assert.NilError(t, err) + + stdOut := strings.TrimSpace(actualStdout.String()) + stdErr := strings.TrimSpace(actualStderr.String()) + if stdOut != tc.stdOut { + t.Fatalf("test produced invalid output: %q, expected %q. Stderr:%q", stdOut, tc.stdOut, stdErr) + } + if stdErr != tc.stdErr { + t.Fatalf("test produced invalid error: %q, expected %q. Stdout:%q", stdErr, tc.stdErr, stdOut) + + } + }) + } + +} diff --git a/integration/capabilities/main_linux_test.go b/integration/capabilities/main_linux_test.go new file mode 100644 index 0000000000000..0f074dd15603e --- /dev/null +++ b/integration/capabilities/main_linux_test.go @@ -0,0 +1,33 @@ +package capabilities + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/testutil/environment" +) + +var testEnv *environment.Execution + +func TestMain(m *testing.M) { + var err error + testEnv, err = environment.New() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + err = environment.EnsureFrozenImagesLinux(testEnv) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + testEnv.Print() + os.Exit(m.Run()) +} + +func setupTest(t *testing.T) func() { + environment.ProtectAll(t, testEnv) + return func() { testEnv.Clean(t) } +} diff --git a/integration/internal/container/ops.go b/integration/internal/container/ops.go index 33d977700623c..838f515e9f24e 100644 --- a/integration/internal/container/ops.go +++ b/integration/internal/container/ops.go @@ -237,3 +237,21 @@ func WithRuntime(name string) func(*TestContainerConfig) { c.HostConfig.Runtime = name } } + +func WithCapability(capabilities ...string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.CapAdd = append(c.HostConfig.CapAdd, capabilities...) + } +} + +func WithDropCapability(capabilities ...string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.CapDrop = append(c.HostConfig.CapDrop, capabilities...) + } +} + +func WithSecurityOpt(opt string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.SecurityOpt = append(c.HostConfig.SecurityOpt, opt) + } +} From 15bd07b4fda2f02eca528e6e437a34bcb4bbef74 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 11 Aug 2023 21:29:53 +0900 Subject: [PATCH 214/293] update runc binary to v1.1.9 Signed-off-by: Akihiro Suda (cherry picked from commit b039bbc678892661e2e42fd69d7aa417887b8ecf) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- hack/dockerfile/install/runc.installer | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4cbee5021570..ef96a665af8ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -280,7 +280,7 @@ RUN git init . && git remote add origin "https://github.com/opencontainers/runc. # that is used. If you need to update runc, open a pull request in the containerd # project first, and update both after that is merged. When updating RUNC_VERSION, # consider updating runc in vendor.mod accordingly. -ARG RUNC_VERSION=v1.1.8 +ARG RUNC_VERSION=v1.1.9 RUN git fetch -q --depth 1 origin "${RUNC_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD FROM base AS runc-build diff --git a/hack/dockerfile/install/runc.installer b/hack/dockerfile/install/runc.installer index 289a0ca9ff478..3ac1ddfbf5f9f 100755 --- a/hack/dockerfile/install/runc.installer +++ b/hack/dockerfile/install/runc.installer @@ -9,7 +9,7 @@ set -e # the containerd project first, and update both after that is merged. # # When updating RUNC_VERSION, consider updating runc in vendor.mod accordingly -: "${RUNC_VERSION:=v1.1.8}" +: "${RUNC_VERSION:=v1.1.9}" install_runc() { RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp"}" From 5d2c383d72f45ec438d39ce52f1eac6490ca660c Mon Sep 17 00:00:00 2001 From: Sam Thibault Date: Mon, 14 Aug 2023 14:03:52 +0200 Subject: [PATCH 215/293] remove s390x and ppc64ls pipelines Signed-off-by: Sam Thibault (cherry picked from commit 59aa3dce8a3abc9d6eb29df7c76b4484e8758234) Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 402 ---------------------------------------------------- 1 file changed, 402 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6707839168792..d93a24a89f08e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,8 +9,6 @@ pipeline { } parameters { booleanParam(name: 'arm64', defaultValue: true, description: 'ARM (arm64) Build/Test') - booleanParam(name: 's390x', defaultValue: false, description: 'IBM Z (s390x) Build/Test') - booleanParam(name: 'ppc64le', defaultValue: false, description: 'PowerPC (ppc64le) Build/Test') booleanParam(name: 'dco', defaultValue: true, description: 'Run the DCO check') } environment { @@ -51,406 +49,6 @@ pipeline { } stage('Build') { parallel { - stage('s390x') { - when { - beforeAgent true - // Skip this stage on PRs unless the checkbox is selected - anyOf { - not { changeRequest() } - expression { params.s390x } - } - } - agent { label 's390x-ubuntu-2004' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker build --force-rm -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Unit tests") { - steps { - sh ''' - sudo modprobe ip6table_filter - ''' - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/test/unit - ''' - } - post { - always { - junit testResults: 'bundles/junit-report*.xml', allowEmptyResults: true - } - } - } - stage("Integration tests") { - environment { TEST_SKIP_INTEGRATION_CLI = '1' } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TESTDEBUG \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ - -e TEST_SKIP_INTEGRATION_CLI \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=s390x-integration - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - stage('s390x integration-cli') { - when { - beforeAgent true - // Skip this stage on PRs unless the checkbox is selected - anyOf { - not { changeRequest() } - expression { params.s390x } - } - } - agent { label 's390x-ubuntu-2004' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker build --force-rm -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Integration-cli tests") { - environment { TEST_SKIP_INTEGRATION = '1' } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ - -e TEST_SKIP_INTEGRATION \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=s390x-integration-cli - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - stage('ppc64le') { - when { - beforeAgent true - // Skip this stage on PRs unless the checkbox is selected - anyOf { - not { changeRequest() } - expression { params.ppc64le } - } - } - agent { label 'ppc64le-ubuntu-1604' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker buildx build --load --force-rm -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Unit tests") { - steps { - sh ''' - sudo modprobe ip6table_filter - ''' - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/test/unit - ''' - } - post { - always { - junit testResults: 'bundles/junit-report*.xml', allowEmptyResults: true - } - } - } - stage("Integration tests") { - environment { TEST_SKIP_INTEGRATION_CLI = '1' } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TESTDEBUG \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ - -e TEST_SKIP_INTEGRATION_CLI \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=ppc64le-integration - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - stage('ppc64le integration-cli') { - when { - beforeAgent true - // Skip this stage on PRs unless the checkbox is selected - anyOf { - not { changeRequest() } - expression { params.ppc64le } - } - } - agent { label 'ppc64le-ubuntu-1604' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker buildx build --load --force-rm -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Integration-cli tests") { - environment { TEST_SKIP_INTEGRATION = '1' } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ - -e TEST_SKIP_INTEGRATION \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=ppc64le-integration-cli - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } stage('arm64') { when { beforeAgent true From 600aa7b7a52a450fc4e1daeceea1636f43b1f6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 16 Aug 2023 11:16:50 +0200 Subject: [PATCH 216/293] c8d/inspect: Ignore manifest with missing config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a failure to inspect image if any of its present manifest references an image config which isn't present locally. Signed-off-by: Paweł Gronowski (cherry picked from commit a64adda4e77890f8b126cf7a723f27735a07e5d2) Signed-off-by: Paweł Gronowski --- daemon/containerd/image.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index 7725476d1fbd1..eaa94b8155da5 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -48,11 +48,25 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima err = i.walkImageManifests(ctx, desc, func(img *ImageManifest) error { conf, err := img.Config(ctx) if err != nil { - return err + if cerrdefs.IsNotFound(err) { + logrus.WithFields(logrus.Fields{ + "manifestDescriptor": img.Target(), + }).Debug("manifest was present, but accessing its config failed, ignoring") + return nil + } + return errdefs.System(fmt.Errorf("failed to get config descriptor: %w", err)) } + var ociimage ocispec.Image if err := readConfig(ctx, cs, conf, &ociimage); err != nil { - return err + if cerrdefs.IsNotFound(err) { + logrus.WithFields(logrus.Fields{ + "manifestDescriptor": img.Target(), + "configDescriptor": conf, + }).Debug("manifest present, but its config is missing, ignoring") + return nil + } + return errdefs.System(fmt.Errorf("failed to read config of the manifest %v: %w", img.Target().Digest, err)) } presentImages = append(presentImages, ociimage) return nil @@ -61,7 +75,8 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima return nil, err } if len(presentImages) == 0 { - return nil, errdefs.NotFound(errors.New("failed to find image manifest")) + ref, _ := reference.ParseAnyReference(refOrID) + return nil, images.ErrImageDoesNotExist{Ref: ref} } sort.SliceStable(presentImages, func(i, j int) bool { From 3ce0dc7e35dcb8466b73e22c7f28e69acd2d8afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 31 Jul 2023 18:32:01 +0200 Subject: [PATCH 217/293] bakefile: Remove default value of DOCKER_GITCOMMIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "HEAD" will still be used as a version if no DOCKER_COMMIT is provided (for example when not running via `make`), but it won't prevent it being set to the GITHUB_SHA variable when it's present. This should fix `Git commit` reported by `docker version` for the binaries generated by `moby-bin`. Signed-off-by: Paweł Gronowski (cherry picked from commit d7a9f15775e023f5675a48381c3d7d78c25dd16d) Signed-off-by: Paweł Gronowski --- docker-bake.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 3d9675f184431..a1ac477a91b08 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -14,7 +14,7 @@ variable "DOCKER_BUILDTAGS" { default = "" } variable "DOCKER_GITCOMMIT" { - default = "HEAD" + default = null } # Docker version such as 23.0.0-dev. Automatically generated through Git ref. @@ -81,7 +81,7 @@ target "_common" { DOCKER_STATIC = DOCKER_STATIC DOCKER_LDFLAGS = DOCKER_LDFLAGS DOCKER_BUILDTAGS = DOCKER_BUILDTAGS - DOCKER_GITCOMMIT = DOCKER_GITCOMMIT != "" ? DOCKER_GITCOMMIT : GITHUB_SHA + DOCKER_GITCOMMIT = DOCKER_GITCOMMIT != null ? DOCKER_GITCOMMIT : GITHUB_SHA VERSION = VERSION != "" ? VERSION : GITHUB_REF PLATFORM = PLATFORM PRODUCT = PRODUCT From 448ae33f875e43ddcd30c8cadfbafb2cbd942637 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Thu, 17 Aug 2023 11:34:38 -0600 Subject: [PATCH 218/293] ci(bin-image): populate DOCKER_GITCOMMIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bjorn Neergaard (cherry picked from commit 9aed6308d42eecabc78900c3c68488b75eff0337) Signed-off-by: Paweł Gronowski --- .github/workflows/bin-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index 182f02c04f999..ffdd25fb6bb7d 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -118,6 +118,8 @@ jobs: ./docker-bake.hcl /tmp/bake-meta.json targets: bin-image + env: + DOCKER_GITCOMMIT: ${{ github.sha }} set: | *.platform=${{ matrix.platform }} *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} From 0c131f58bae16688926e0af72790dfec22ec3b4b Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Thu, 17 Aug 2023 13:17:33 -0600 Subject: [PATCH 219/293] ci(bin-image): populate DOCKER_GITCOMMIT, take 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bjorn Neergaard (cherry picked from commit 73ffb48bfbbc1fb927c689914cfff5992129da0f) Signed-off-by: Paweł Gronowski --- .github/workflows/bin-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index ffdd25fb6bb7d..aa6d41130cdcb 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -113,13 +113,13 @@ jobs: name: Build id: bake uses: docker/bake-action@v3 + env: + DOCKER_GITCOMMIT: ${{ github.sha }} with: files: | ./docker-bake.hcl /tmp/bake-meta.json targets: bin-image - env: - DOCKER_GITCOMMIT: ${{ github.sha }} set: | *.platform=${{ matrix.platform }} *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} From 3897724f4a7c036b3b1b7ecb2feb66056696f6e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 18 Aug 2023 11:38:50 +0200 Subject: [PATCH 220/293] volume/local: Fix debug log typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Active count is incremented, but message claimed the opposite. Signed-off-by: Paweł Gronowski (cherry picked from commit 7f965d55c719b8777221c4dbf9a1eb93cfada8ab) Signed-off-by: Paweł Gronowski --- volume/local/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volume/local/local.go b/volume/local/local.go index f156ea2339bc3..1417941cca3a7 100644 --- a/volume/local/local.go +++ b/volume/local/local.go @@ -310,7 +310,7 @@ func (v *localVolume) Mount(id string) (string, error) { v.active.mounted = true } v.active.count++ - logger.WithField("active mounts", v.active).Debug("Decremented active mount count") + logger.WithField("active mounts", v.active).Debug("Incremented active mount count") } if err := v.postMount(); err != nil { return "", err From 54953f2f5a5157a9ae92d25831a2b77c20cfab82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 10 Aug 2023 19:13:38 +0200 Subject: [PATCH 221/293] integration: Add test for not breaking overlayfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check that operations that could potentially perform overlayfs mounts that could cause undefined behaviors. Signed-off-by: Paweł Gronowski (cherry picked from commit 303e2b124e6697a232d0c1a5207cddd79e561fe1) Signed-off-by: Paweł Gronowski --- integration/container/overlayfs_linux_test.go | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 integration/container/overlayfs_linux_test.go diff --git a/integration/container/overlayfs_linux_test.go b/integration/container/overlayfs_linux_test.go new file mode 100644 index 0000000000000..4736e6afc22f1 --- /dev/null +++ b/integration/container/overlayfs_linux_test.go @@ -0,0 +1,111 @@ +package container + +import ( + "context" + "io" + "strings" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/dmesg" + "gotest.tools/v3/assert" + "gotest.tools/v3/skip" +) + +func TestNoOverlayfsWarningsAboutUndefinedBehaviors(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType != "linux", "overlayfs is only available on linux") + skip.If(t, testEnv.IsRemoteDaemon(), "local daemon is needed for kernel log access") + skip.If(t, testEnv.IsRootless(), "root is needed for reading kernel log") + + defer setupTest(t)() + client := testEnv.APIClient() + ctx := context.Background() + + cID := container.Run(ctx, t, client, container.WithCmd("sh", "-c", `while true; do echo $RANDOM >>/file; sleep 0.1; done`)) + + testCases := []struct { + name string + operation func(t *testing.T) error + }{ + {name: "diff", operation: func(*testing.T) error { + _, err := client.ContainerDiff(ctx, cID) + return err + }}, + {name: "export", operation: func(*testing.T) error { + rc, err := client.ContainerExport(ctx, cID) + if err == nil { + defer rc.Close() + _, err = io.Copy(io.Discard, rc) + } + return err + }}, + {name: "cp to container", operation: func(t *testing.T) error { + archive, err := archive.Generate("new-file", "hello-world") + assert.NilError(t, err, "failed to create a temporary archive") + return client.CopyToContainer(ctx, cID, "/", archive, types.CopyToContainerOptions{}) + }}, + {name: "cp from container", operation: func(*testing.T) error { + rc, _, err := client.CopyFromContainer(ctx, cID, "/file") + if err == nil { + defer rc.Close() + _, err = io.Copy(io.Discard, rc) + } + + return err + }}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + prev := dmesgLines(256) + + err := tc.operation(t) + assert.NilError(t, err) + + after := dmesgLines(2048) + + diff := diffDmesg(prev, after) + for _, line := range diff { + overlayfs := strings.Contains(line, "overlayfs: ") + lowerDirInUse := strings.Contains(line, "lowerdir is in-use as ") + upperDirInUse := strings.Contains(line, "upperdir is in-use as ") + workDirInuse := strings.Contains(line, "workdir is in-use as ") + undefinedBehavior := strings.Contains(line, "will result in undefined behavior") + + if overlayfs && (lowerDirInUse || upperDirInUse || workDirInuse) && undefinedBehavior { + t.Errorf("%s caused overlayfs kernel warning: %s", tc.name, line) + } + } + }) + } +} + +func dmesgLines(bytes int) []string { + data := dmesg.Dmesg(bytes) + return strings.Split(strings.TrimSpace(string(data)), "\n") +} + +func diffDmesg(prev, next []string) []string { + // All lines have a timestamp, so just take the last one from the previous + // log and find it in the new log. + lastPrev := prev[len(prev)-1] + + for idx := len(next) - 1; idx >= 0; idx-- { + line := next[idx] + + if line == lastPrev { + nextIdx := idx + 1 + if nextIdx < len(next) { + return next[nextIdx:] + } else { + // Found at the last position, log is the same. + return nil + } + } + } + + return next +} From b76a0c7d009c9859eb01fce680685cb55b4a36bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 10 Aug 2023 15:33:06 +0200 Subject: [PATCH 222/293] c8d/export: Use ref counted mounter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To prevent mounting the container rootfs in a rw mode if it's already mounted. This can't use `mount.WithReadonlyTempMount` because the archive code does a chroot with a pivot_root, which creates a new directory in the rootfs. Signed-off-by: Paweł Gronowski (cherry picked from commit 051d51b22212bb570998e8cf3593036177c4a647) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_exporter.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/daemon/containerd/image_exporter.go b/daemon/containerd/image_exporter.go index 21bbcfc48ef4c..9dfb3e6a0ec73 100644 --- a/daemon/containerd/image_exporter.go +++ b/daemon/containerd/image_exporter.go @@ -11,7 +11,6 @@ import ( containerdimages "github.com/containerd/containerd/images" "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/leases" - "github.com/containerd/containerd/mount" cplatforms "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" "github.com/docker/docker/container" @@ -30,7 +29,13 @@ func (i *ImageService) PerformWithBaseFS(ctx context.Context, c *container.Conta if err != nil { return err } - return mount.WithTempMount(ctx, mounts, fn) + path, err := i.refCountMounter.Mount(mounts, c.ID) + if err != nil { + return err + } + defer i.refCountMounter.Unmount(path) + + return fn(path) } // ExportImage exports a list of images to the given output stream. The From 74bf46aea6e17fe088e5efa6647c4fb558b22398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 10 Aug 2023 19:05:03 +0200 Subject: [PATCH 223/293] c8d/diff: Reuse mount, mount parent as read-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The container rw layer may already be mounted, so it's not safe to use it in another overlay mount. Use the ref counted mounter (which will reuse the existing mount if it exists) to avoid that. Also, mount the parent mounts (layers of the base image) in a read-only mode. Signed-off-by: Paweł Gronowski (cherry picked from commit 6da42ca8308fccf3ae5be75d599eeda61f7e41ed) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_changes.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/daemon/containerd/image_changes.go b/daemon/containerd/image_changes.go index 94c56cf6161c6..dd44aebe80717 100644 --- a/daemon/containerd/image_changes.go +++ b/daemon/containerd/image_changes.go @@ -58,15 +58,10 @@ func (i *ImageService) Changes(ctx context.Context, container *container.Contain } }() - mounts, err := snapshotter.Mounts(ctx, container.ID) - if err != nil { - return nil, err - } - var changes []archive.Change - err = mount.WithReadonlyTempMount(ctx, mounts, func(fs string) error { - return mount.WithTempMount(ctx, parent, func(root string) error { - changes, err = archive.ChangesDirs(fs, root) + err = i.PerformWithBaseFS(ctx, container, func(containerRootfs string) error { + return mount.WithReadonlyTempMount(ctx, parent, func(parentRootfs string) error { + changes, err = archive.ChangesDirs(containerRootfs, parentRootfs) return err }) }) From fb6784bdf0bdbc3ff59052632f3121e7c556d697 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 21 Aug 2023 18:51:41 +0200 Subject: [PATCH 224/293] gha: set 10-minute timeout on "report" actions I had a CI run fail to "Upload reports": Exponential backoff for retry #1. Waiting for 4565 milliseconds before continuing the upload at offset 0 Finished backoff for retry #1, continuing with upload Total file count: 211 ---- Processed file #160 (75.8%) ... Total file count: 211 ---- Processed file #164 (77.7%) Total file count: 211 ---- Processed file #164 (77.7%) Total file count: 211 ---- Processed file #164 (77.7%) A 503 status code has been received, will attempt to retry the upload ##### Begin Diagnostic HTTP information ##### Status Code: 503 Status Message: Service Unavailable Header Information: { "content-length": "592", "content-type": "application/json; charset=utf-8", "date": "Mon, 21 Aug 2023 14:08:10 GMT", "server": "Kestrel", "cache-control": "no-store,no-cache", "pragma": "no-cache", "strict-transport-security": "max-age=2592000", "x-tfs-processid": "b2fc902c-011a-48be-858d-c62e9c397cb6", "activityid": "49a48b53-0411-4ff3-86a7-4528e3f71ba2", "x-tfs-session": "49a48b53-0411-4ff3-86a7-4528e3f71ba2", "x-vss-e2eid": "49a48b53-0411-4ff3-86a7-4528e3f71ba2", "x-vss-senderdeploymentid": "63be6134-28d1-8c82-e969-91f4e88fcdec", "x-frame-options": "SAMEORIGIN" } ###### End Diagnostic HTTP information ###### Retry limit has been reached for chunk at offset 0 to https://pipelinesghubeus5.actions.githubusercontent.com/Y2huPMnV2RyiTvKoReSyXTCrcRyxUdSDRZYoZr0ONBvpl5e9Nu/_apis/resources/Containers/8331549?itemPath=integration-reports%2Fubuntu-22.04-systemd%2Fbundles%2Ftest-integration%2FTestInfoRegistryMirrors%2Fd20ac12e48cea%2Fdocker.log Warning: Aborting upload for /tmp/reports/ubuntu-22.04-systemd/bundles/test-integration/TestInfoRegistryMirrors/d20ac12e48cea/docker.log due to failure Error: aborting artifact upload Total file count: 211 ---- Processed file #165 (78.1%) A 503 status code has been received, will attempt to retry the upload Exponential backoff for retry #1. Waiting for 5799 milliseconds before continuing the upload at offset 0 As a result, the "Download reports" continued retrying: ... Total file count: 1004 ---- Processed file #436 (43.4%) Total file count: 1004 ---- Processed file #436 (43.4%) Total file count: 1004 ---- Processed file #436 (43.4%) An error occurred while attempting to download a file Error: Request timeout: /Y2huPMnV2RyiTvKoReSyXTCrcRyxUdSDRZYoZr0ONBvpl5e9Nu/_apis/resources/Containers/8331549?itemPath=integration-reports%2Fubuntu-20.04%2Fbundles%2Ftest-integration%2FTestCreateWithDuplicateNetworkNames%2Fd47798cc212d1%2Fdocker.log at ClientRequest. (/home/runner/work/_actions/actions/download-artifact/v3/dist/index.js:3681:26) at Object.onceWrapper (node:events:627:28) at ClientRequest.emit (node:events:513:28) at TLSSocket.emitRequestTimeout (node:_http_client:839:9) at Object.onceWrapper (node:events:627:28) at TLSSocket.emit (node:events:525:35) at TLSSocket.Socket._onTimeout (node:net:550:8) at listOnTimeout (node:internal/timers:559:17) at processTimers (node:internal/timers:502:7) Exponential backoff for retry #1. Waiting for 5305 milliseconds before continuing the download Total file count: 1004 ---- Processed file #436 (43.4%) And, it looks like GitHub doesn't allow cancelling the job, possibly because it is defined with `if: always()`? Signed-off-by: Sebastiaan van Stijn (cherry picked from commit d6f340e784ee2b4e208be9834f75f042a0fb6691) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e7467f418c75..73d3092b7f52b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,6 +166,7 @@ jobs: unit-report: runs-on: ubuntu-20.04 + timeout-minutes: 10 if: always() needs: - unit @@ -354,6 +355,7 @@ jobs: integration-report: runs-on: ubuntu-20.04 + timeout-minutes: 10 if: always() needs: - integration @@ -482,6 +484,7 @@ jobs: integration-cli-report: runs-on: ubuntu-20.04 + timeout-minutes: 10 if: always() needs: - integration-cli From bb22b8a41876ab3df411501994199c0cd21f7ed5 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 19 Aug 2023 07:19:10 +0200 Subject: [PATCH 225/293] ci(bin-image): check repo origin Signed-off-by: CrazyMax (cherry picked from commit 219d4d9db9941e965c21eca045b7472d43b5599c) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/bin-image.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index aa6d41130cdcb..19286d226a90c 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -104,7 +104,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.repository == 'moby/moby' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_MOBYBIN_USERNAME }} @@ -122,18 +122,18 @@ jobs: targets: bin-image set: | *.platform=${{ matrix.platform }} - *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} + *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' && github.repository == 'moby/moby' }} *.tags= - name: Export digest - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.repository == 'moby/moby' run: | mkdir -p /tmp/digests digest="${{ fromJSON(steps.bake.outputs.metadata)['bin-image']['containerimage.digest'] }}" touch "/tmp/digests/${digest#sha256:}" - name: Upload digest - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.repository == 'moby/moby' uses: actions/upload-artifact@v3 with: name: digests @@ -143,7 +143,7 @@ jobs: merge: runs-on: ubuntu-20.04 - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.repository == 'moby/moby' needs: - build steps: From e23979958366b0157257f7baab23fce19b5bc958 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 31 Jul 2023 16:14:43 +0200 Subject: [PATCH 226/293] distribution: update warning for deprecated image formats - Use the same warning for both "v1 in manifest-index" and bare "v1" images. - Update URL to use a "/go/" redirect, which allows the docs team to more easily redirect the URL to relevant docs (if things move). Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 982bc0e228e89c3117cc326301ff5b4b9bef3a8a) Signed-off-by: Sebastiaan van Stijn --- distribution/pull_v2.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index 06e10a9b8d54b..2277d4e58fdc4 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -441,7 +441,7 @@ func (p *puller) pullTag(ctx context.Context, ref reference.Named, platform *oci // give registries time to upgrade to schema2 and only warn if we know a registry has been upgraded long time ago // TODO: condition to be removed if reference.Domain(ref) == "docker.io" { - msg := fmt.Sprintf("Image %s uses outdated schema1 manifest format. Please upgrade to a schema2 image for better future compatibility. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/", ref) + msg := fmt.Sprintf("[DEPRECATION NOTICE] Docker Image Format v1, and Docker Image manifest version 2, schema 1 support will be removed in an upcoming release. Suggest the author of %s to upgrade the image to the OCI Format, or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/", ref) logrus.Warn(msg) progress.Message(p.config.ProgressOutput, "", msg) } @@ -873,7 +873,7 @@ func (p *puller) pullManifestList(ctx context.Context, ref reference.Named, mfst switch v := manifest.(type) { case *schema1.SignedManifest: - msg := fmt.Sprintf("[DEPRECATION NOTICE] v2 schema1 manifests in manifest lists are not supported and will break in a future release. Suggest author of %s to upgrade to v2 schema2. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/", ref) + msg := fmt.Sprintf("[DEPRECATION NOTICE] Docker Image Format v1, and Docker Image manifest version 2, schema 1 support will be removed in an upcoming release. Suggest the author of %s to upgrade the image to the OCI Format, or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/", ref) logrus.Warn(msg) progress.Message(p.config.ProgressOutput, "", msg) From a99e62fa3d79bc1b4b9b4aaafbd71655b3aa6397 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 1 Aug 2023 21:03:07 +0200 Subject: [PATCH 227/293] distribution: show image schema deprecation on all registries When we added this deprecation warning, some registries had not yet moved away from the deprecated specification, so we made the warning conditional for pulling from Docker Hub. That condition was added in 647dfe99a50badd27f0508c67eddc4b4923fcef7, which is over 4 Years ago, which should be time enough for images and registries to have moved to current specifications. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 8c4af5dacb2232f86c33defc0d45fcdc8af80ea0) Signed-off-by: Sebastiaan van Stijn --- distribution/pull_v2.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index 2277d4e58fdc4..b83e8defd2b28 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -438,13 +438,9 @@ func (p *puller) pullTag(ctx context.Context, ref reference.Named, platform *oci switch v := manifest.(type) { case *schema1.SignedManifest: - // give registries time to upgrade to schema2 and only warn if we know a registry has been upgraded long time ago - // TODO: condition to be removed - if reference.Domain(ref) == "docker.io" { - msg := fmt.Sprintf("[DEPRECATION NOTICE] Docker Image Format v1, and Docker Image manifest version 2, schema 1 support will be removed in an upcoming release. Suggest the author of %s to upgrade the image to the OCI Format, or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/", ref) - logrus.Warn(msg) - progress.Message(p.config.ProgressOutput, "", msg) - } + msg := fmt.Sprintf("[DEPRECATION NOTICE] Docker Image Format v1, and Docker Image manifest version 2, schema 1 support will be removed in an upcoming release. Suggest the author of %s to upgrade the image to the OCI Format, or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/", ref) + logrus.Warn(msg) + progress.Message(p.config.ProgressOutput, "", msg) id, manifestDigest, err = p.pullSchema1(ctx, ref, v, platform) if err != nil { From aaf84dd4cf88fd17f8d8af22395e4acccd667b6f Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 25 Nov 2022 10:37:05 +0100 Subject: [PATCH 228/293] remove Dockerfile.e2e Dockerfile.e2e is not used anymore. Integration tests run through the main Dockerfile. Also removes the daemon OS/Arch detection script that is not necessary anymore. It was used to select the Dockerfile based on the arch like Dockerfile.arm64 but we don't have those anymore. Was also used to check referenced frozen images in the Dockerfile. Signed-off-by: CrazyMax (cherry picked from commit 5efe72415d975ec0d4e097cfc235bd6c6c62418b) Signed-off-by: Bjorn Neergaard --- Dockerfile.e2e | 84 ------------------------- Makefile | 6 +- hack/make/.detect-daemon-osarch | 43 ------------- hack/make/.integration-daemon-setup | 7 --- hack/make/.integration-daemon-start | 4 ++ hack/make/.integration-test-helpers | 1 - hack/make/build-integration-test-binary | 7 --- hack/make/test-integration | 1 - hack/make/test-integration-cli | 6 -- hack/make/test-integration-shell | 1 - hack/test/e2e-run.sh | 1 - 11 files changed, 5 insertions(+), 156 deletions(-) delete mode 100644 Dockerfile.e2e delete mode 100644 hack/make/.detect-daemon-osarch delete mode 100644 hack/make/.integration-daemon-setup delete mode 100755 hack/make/build-integration-test-binary delete mode 100755 hack/make/test-integration-cli diff --git a/Dockerfile.e2e b/Dockerfile.e2e deleted file mode 100644 index 2b41a7e65e4de..0000000000000 --- a/Dockerfile.e2e +++ /dev/null @@ -1,84 +0,0 @@ -ARG GO_VERSION=1.20.4 - -FROM golang:${GO_VERSION}-alpine AS base -ENV GO111MODULE=off -RUN apk --no-cache add \ - bash \ - build-base \ - curl \ - lvm2-dev \ - jq - -RUN mkdir -p /build/ -RUN mkdir -p /go/src/github.com/docker/docker/ -WORKDIR /go/src/github.com/docker/docker/ - -FROM base AS frozen-images -# Get useful and necessary Hub images so we can "docker load" locally instead of pulling -COPY contrib/download-frozen-image-v2.sh / -RUN /download-frozen-image-v2.sh /build \ - busybox:latest@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 \ - busybox:latest@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 \ - debian:bullseye-slim@sha256:dacf278785a4daa9de07596ec739dbc07131e189942772210709c5c0777e8437 \ - hello-world:latest@sha256:d58e752213a51785838f9eed2b7a498ffa1cb3aa7f946dda11af39286c3db9a9 \ - arm32v7/hello-world:latest@sha256:50b8560ad574c779908da71f7ce370c0a2471c098d44d1c8f6b513c5a55eeeb1 -# See also frozenImages in "testutil/environment/protect.go" (which needs to be updated when adding images to this list) - -FROM base AS dockercli -COPY hack/dockerfile/install/install.sh ./install.sh -COPY hack/dockerfile/install/dockercli.installer ./ -RUN PREFIX=/build ./install.sh dockercli - -# TestDockerCLIBuildSuite dependency -FROM base AS contrib -COPY contrib/syscall-test /build/syscall-test -COPY contrib/httpserver/Dockerfile /build/httpserver/Dockerfile -COPY contrib/httpserver contrib/httpserver -RUN CGO_ENABLED=0 go build -buildmode=pie -o /build/httpserver/httpserver github.com/docker/docker/contrib/httpserver - -# Build the integration tests and copy the resulting binaries to /build/tests -FROM base AS builder - -# Set tag and add sources -COPY . . -# Copy test sources tests that use assert can print errors -RUN mkdir -p /build${PWD} && find integration integration-cli -name \*_test.go -exec cp --parents '{}' /build${PWD} \; -# Build and install test binaries -ARG DOCKER_GITCOMMIT=undefined -RUN hack/make.sh build-integration-test-binary -RUN mkdir -p /build/tests && find . -name test.main -exec cp --parents '{}' /build/tests \; - -## Generate testing image -FROM alpine:3.10 as runner - -ENV DOCKER_REMOTE_DAEMON=1 -ENV DOCKER_INTEGRATION_DAEMON_DEST=/ -ENTRYPOINT ["/scripts/run.sh"] - -# Add an unprivileged user to be used for tests which need it -RUN addgroup docker && adduser -D -G docker unprivilegeduser -s /bin/ash - -# GNU tar is used for generating the emptyfs image -RUN apk --no-cache add \ - bash \ - ca-certificates \ - g++ \ - git \ - inetutils-ping \ - iptables \ - libcap2-bin \ - pigz \ - tar \ - xz - -COPY hack/test/e2e-run.sh /scripts/run.sh -COPY hack/make/.build-empty-images /scripts/build-empty-images.sh - -COPY integration/testdata /tests/integration/testdata -COPY integration/build/testdata /tests/integration/build/testdata -COPY integration-cli/fixtures /tests/integration-cli/fixtures - -COPY --from=frozen-images /build/ /docker-frozen-images -COPY --from=dockercli /build/ /usr/bin/ -COPY --from=contrib /build/ /tests/contrib/ -COPY --from=builder /build/ / diff --git a/Makefile b/Makefile index ff7f713466de7..789bfc2b4eea7 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,6 @@ BUILDX ?= $(DOCKER) buildx DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) export DOCKER_GRAPHDRIVER -# get OS/Arch of docker engine -DOCKER_OSARCH := $(shell bash -c 'source hack/make/.detect-daemon-osarch && echo $${DOCKER_ENGINE_OSARCH}') -DOCKERFILE := $(shell bash -c 'source hack/make/.detect-daemon-osarch && echo $${DOCKERFILE}') - DOCKER_GITCOMMIT := $(shell git rev-parse --short HEAD || echo unsupported) export DOCKER_GITCOMMIT @@ -150,7 +146,7 @@ ifdef DOCKER_SYSTEMD DOCKER_BUILD_ARGS += --build-arg=SYSTEMD=true endif -BUILD_OPTS := ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -f "$(DOCKERFILE)" +BUILD_OPTS := ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} BUILD_CMD := $(BUILDX) build BAKE_CMD := $(BUILDX) bake diff --git a/hack/make/.detect-daemon-osarch b/hack/make/.detect-daemon-osarch deleted file mode 100644 index 9190cd0264bae..0000000000000 --- a/hack/make/.detect-daemon-osarch +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -e - -docker-version-osarch() { - if ! type docker &> /dev/null; then - # docker is not installed - return - fi - local target="$1" # "Client" or "Server" - local fmtStr="{{.${target}.Os}}/{{.${target}.Arch}}" - if docker version -f "$fmtStr" 2> /dev/null; then - # if "docker version -f" works, let's just use that! - return - fi - docker version | awk ' - $1 ~ /^(Client|Server):$/ { section = 0 } - $1 == "'"$target"':" { section = 1; next } - section && $1 == "OS/Arch:" { print $2 } - - # old versions of Docker - $1 == "OS/Arch" && $2 == "('"${target,,}"'):" { print $3 } - ' -} - -# Retrieve OS/ARCH of docker daemon, e.g. linux/amd64 -export DOCKER_ENGINE_OSARCH="${DOCKER_ENGINE_OSARCH:=$(docker-version-osarch 'Server')}" -export DOCKER_ENGINE_GOOS="${DOCKER_ENGINE_OSARCH%/*}" -export DOCKER_ENGINE_GOARCH="${DOCKER_ENGINE_OSARCH##*/}" -DOCKER_ENGINE_GOARCH=${DOCKER_ENGINE_GOARCH:=amd64} - -# and the client, just in case -export DOCKER_CLIENT_OSARCH="$(docker-version-osarch 'Client')" -export DOCKER_CLIENT_GOOS="${DOCKER_CLIENT_OSARCH%/*}" -export DOCKER_CLIENT_GOARCH="${DOCKER_CLIENT_OSARCH##*/}" -DOCKER_CLIENT_GOARCH=${DOCKER_CLIENT_GOARCH:=amd64} - -DOCKERFILE='Dockerfile' - -if [ "${DOCKER_ENGINE_GOOS:-$DOCKER_CLIENT_GOOS}" = "windows" ]; then - DOCKERFILE='Dockerfile.windows' -fi - -export DOCKERFILE diff --git a/hack/make/.integration-daemon-setup b/hack/make/.integration-daemon-setup deleted file mode 100644 index 4bcc816c2c3e8..0000000000000 --- a/hack/make/.integration-daemon-setup +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -e - -source "$MAKEDIR/.detect-daemon-osarch" -if [ "$DOCKER_ENGINE_GOOS" != "windows" ]; then - bundle .build-empty-images -fi diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index 89e38993dbbe1..04934d7b5a8b1 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -151,3 +151,7 @@ while ! ${TEST_CLIENT_BINARY} version &> /dev/null; do sleep 2 done printf "\n" + +if [ "$(docker version --format '{{ .Server.Os }}')" != 'windows' ]; then + bundle .build-empty-images +fi diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index e9062ac66cdec..16bab622c104b 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -181,7 +181,6 @@ test_env() { DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \ DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \ DOCKER_ROOTLESS="$DOCKER_ROOTLESS" \ - DOCKERFILE="$DOCKERFILE" \ GITHUB_ACTIONS="$GITHUB_ACTIONS" \ GOCACHE="$GOCACHE" \ GOPATH="$GOPATH" \ diff --git a/hack/make/build-integration-test-binary b/hack/make/build-integration-test-binary deleted file mode 100755 index 698717f0f5640..0000000000000 --- a/hack/make/build-integration-test-binary +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -# required by https://github.com/AkihiroSuda/kube-moby-integration -set -e - -source hack/make/.integration-test-helpers - -build_test_suite_binaries diff --git a/hack/make/test-integration b/hack/make/test-integration index 199131c209ca2..5cfbc89697c4d 100755 --- a/hack/make/test-integration +++ b/hack/make/test-integration @@ -12,7 +12,6 @@ fi env build_test_suite_binaries bundle .integration-daemon-start - bundle .integration-daemon-setup testexit=0 (repeat run_test_integration) || testexit=$? diff --git a/hack/make/test-integration-cli b/hack/make/test-integration-cli deleted file mode 100755 index 480851e70f487..0000000000000 --- a/hack/make/test-integration-cli +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e -echo "WARNING: test-integration-cli is DEPRECATED. Use test-integration." >&2 - -# TODO: remove this and exit 1 once CI has changed to use test-integration -bundle test-integration diff --git a/hack/make/test-integration-shell b/hack/make/test-integration-shell index bcfa4682eb324..1ee23b3806054 100644 --- a/hack/make/test-integration-shell +++ b/hack/make/test-integration-shell @@ -1,7 +1,6 @@ #!/usr/bin/env bash bundle .integration-daemon-start -bundle .integration-daemon-setup export ABS_DEST bash +e diff --git a/hack/test/e2e-run.sh b/hack/test/e2e-run.sh index 545504fa0e7c8..5e4d8f2154b88 100755 --- a/hack/test/e2e-run.sh +++ b/hack/test/e2e-run.sh @@ -59,7 +59,6 @@ test_env() { DOCKER_HOST="$DOCKER_HOST" \ DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \ DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \ - DOCKERFILE="$DOCKERFILE" \ GOPATH="$GOPATH" \ GOTRACEBACK=all \ HOME="$ABS_DEST/fake-HOME" \ From 5eef5a7f59a069152f3debfd8a55f9c0b078d253 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 18 Aug 2023 08:00:02 -0600 Subject: [PATCH 229/293] ci(bin-image): clean up env var handling There are still messy special cases (e.g. DOCKER_GITCOMMIT vs VERSION), but this makes things a little easier to follow, as we keep GHA-specifics in the GHA files. Signed-off-by: Bjorn Neergaard (cherry picked from commit ad91fc1b0014a68efc660243288f4e65111794de) Signed-off-by: Bjorn Neergaard --- .github/workflows/bin-image.yml | 4 ++-- docker-bake.hcl | 16 ++-------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index 19286d226a90c..b5fce120bf0aa 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -16,6 +16,8 @@ on: env: MOBYBIN_REPO_SLUG: moby/moby-bin + DOCKER_GITCOMMIT: ${{ github.sha }} + VERSION: ${{ github.ref }} PLATFORM: Moby Engine PRODUCT: Moby DEFAULT_PRODUCT_LICENSE: Moby @@ -113,8 +115,6 @@ jobs: name: Build id: bake uses: docker/bake-action@v3 - env: - DOCKER_GITCOMMIT: ${{ github.sha }} with: files: | ./docker-bake.hcl diff --git a/docker-bake.hcl b/docker-bake.hcl index a1ac477a91b08..5666f83d94762 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -47,18 +47,6 @@ variable "PACKAGER_NAME" { default = "" } -# GITHUB_REF is the actual ref that triggers the workflow and used as version -# when tag is pushed: https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables -variable "GITHUB_REF" { - default = "" -} - -# GITHUB_SHA is the commit SHA that triggered the workflow and used as commit. -# https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables -variable "GITHUB_SHA" { - default = "" -} - # Special target: https://github.com/docker/metadata-action#bake-definition target "docker-metadata-action" { tags = ["moby-bin:local"] @@ -81,8 +69,8 @@ target "_common" { DOCKER_STATIC = DOCKER_STATIC DOCKER_LDFLAGS = DOCKER_LDFLAGS DOCKER_BUILDTAGS = DOCKER_BUILDTAGS - DOCKER_GITCOMMIT = DOCKER_GITCOMMIT != null ? DOCKER_GITCOMMIT : GITHUB_SHA - VERSION = VERSION != "" ? VERSION : GITHUB_REF + DOCKER_GITCOMMIT = DOCKER_GITCOMMIT + VERSION = VERSION PLATFORM = PLATFORM PRODUCT = PRODUCT DEFAULT_PRODUCT_LICENSE = DEFAULT_PRODUCT_LICENSE From ac2a80fcc3ef348b20dfb77f2703543d9ee4c7a2 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 18 Aug 2023 08:09:55 -0600 Subject: [PATCH 230/293] ci(bin-image): clean up metadata Signed-off-by: Bjorn Neergaard (cherry picked from commit 2010f4338e2054177b364e727878bfadd2e68d5c) Signed-off-by: Bjorn Neergaard --- .github/workflows/bin-image.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index b5fce120bf0aa..17688ea652132 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -18,10 +18,9 @@ env: MOBYBIN_REPO_SLUG: moby/moby-bin DOCKER_GITCOMMIT: ${{ github.sha }} VERSION: ${{ github.ref }} - PLATFORM: Moby Engine - PRODUCT: Moby - DEFAULT_PRODUCT_LICENSE: Moby - PACKAGER_NAME: Moby + PLATFORM: Moby Engine - Nightly + PRODUCT: moby-bin + PACKAGER_NAME: The Moby Project jobs: validate-dco: From 4ac2355d6274fe195e9b9f93200b362e48f91713 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 18 Aug 2023 08:15:30 -0600 Subject: [PATCH 231/293] hack: use long SHA for DOCKER_GITCOMMIT This better aligns to GHA/CI settings, and is in general a better practice in the year 2023. We also drop the 'unsupported' fallback for `git rev-parse` in the Makefile; we have a better fallback behavior for an empty DOCKER_GITCOMMIT in `hack/make.sh`. Signed-off-by: Bjorn Neergaard (cherry picked from commit d125823d3fef096013912353f723b620f34d038a) Signed-off-by: Bjorn Neergaard --- Makefile | 2 +- hack/make.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 789bfc2b4eea7..8693ea307981e 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ BUILDX ?= $(DOCKER) buildx DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) export DOCKER_GRAPHDRIVER -DOCKER_GITCOMMIT := $(shell git rev-parse --short HEAD || echo unsupported) +DOCKER_GITCOMMIT := $(shell git rev-parse HEAD) export DOCKER_GITCOMMIT # allow overriding the repository and branch that validation scripts are running diff --git a/hack/make.sh b/hack/make.sh index 189ee5aa8b170..d8c7667cda4be 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -51,7 +51,7 @@ fi if [ "$DOCKER_GITCOMMIT" ]; then GITCOMMIT="$DOCKER_GITCOMMIT" elif command -v git &> /dev/null && [ -e .git ] && git rev-parse &> /dev/null; then - GITCOMMIT=$(git rev-parse --short HEAD) + GITCOMMIT=$(git rev-parse HEAD) if [ -n "$(git status --porcelain --untracked-files=no)" ]; then GITCOMMIT="$GITCOMMIT-unsupported" echo "#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -66,8 +66,8 @@ elif command -v git &> /dev/null && [ -e .git ] && git rev-parse &> /dev/null; t else echo >&2 'error: .git directory missing and DOCKER_GITCOMMIT not specified' echo >&2 ' Please either build with the .git directory accessible, or specify the' - echo >&2 ' exact (--short) commit hash you are building using DOCKER_GITCOMMIT for' - echo >&2 ' future accountability in diagnosing build issues. Thanks!' + echo >&2 ' exact commit hash you are building using DOCKER_GITCOMMIT for future' + echo >&2 ' accountability in diagnosing build issues. Thanks!' exit 1 fi From b83f5a89f481ab5b06816932cb8b2b09e5f62c50 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Tue, 22 Aug 2023 11:45:27 +0200 Subject: [PATCH 232/293] Don't return an error if the lease is not found If the image for the wanted platform doesn't exist then the lease doesn't exist either. Returning this error hides the real error, so let's not return it. Signed-off-by: Djordje Lukic (cherry picked from commit b8ff8ea58ee37d672f42e94da4d73442d8a81fc9) Signed-off-by: Sebastiaan van Stijn --- daemon/images/image.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/daemon/images/image.go b/daemon/images/image.go index a773bd0cd2cbc..553f3c46dd50b 100644 --- a/daemon/images/image.go +++ b/daemon/images/image.go @@ -54,10 +54,13 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, image str func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.Image, platform ocispec.Platform) (bool, error) { logger := logrus.WithField("image", img.ID).WithField("desiredPlatform", platforms.Format(platform)) - ls, leaseErr := i.leases.ListResources(ctx, leases.Lease{ID: imageKey(img.ID().String())}) - if leaseErr != nil { - logger.WithError(leaseErr).Error("Error looking up image leases") - return false, leaseErr + ls, err := i.leases.ListResources(ctx, leases.Lease{ID: imageKey(img.ID().String())}) + if err != nil { + if cerrdefs.IsNotFound(err) { + return false, nil + } + logger.WithError(err).Error("Error looking up image leases") + return false, err } // Note we are comparing against manifest lists here, which we expect to always have a CPU variant set (where applicable). From 49671250f6d6f91537a0decb44164b4b34d74b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 21 Aug 2023 16:50:07 +0200 Subject: [PATCH 233/293] c8d/commit: Don't produce an empty layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the diff is empty and don't produce an empty layer. Signed-off-by: Paweł Gronowski (cherry picked from commit eb56493f4ea0fe3e02f3dbf59631cb1a13aa5c63) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/image_commit.go | 89 ++++++++++++++++++++----------- pkg/archive/diff.go | 19 +++++++ 2 files changed, 76 insertions(+), 32 deletions(-) diff --git a/daemon/containerd/image_commit.go b/daemon/containerd/image_commit.go index 56941ee58625c..077b53efa3c21 100644 --- a/daemon/containerd/image_commit.go +++ b/daemon/containerd/image_commit.go @@ -20,6 +20,7 @@ import ( "github.com/containerd/containerd/snapshots" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/image" + "github.com/docker/docker/pkg/archive" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" "github.com/opencontainers/image-spec/specs-go" @@ -38,28 +39,27 @@ func (i *ImageService) CommitImage(ctx context.Context, cc backend.CommitConfig) container := i.containers.Get(cc.ContainerID) cs := i.client.ContentStore() - imageManifest, err := getContainerImageManifest(container) - if err != nil { - return "", err - } + var parentManifest ocispec.Manifest + var parentImage ocispec.Image - imageManifestBytes, err := content.ReadBlob(ctx, cs, imageManifest) - if err != nil { - return "", err - } + // ImageManifest can be nil when committing an image with base FROM scratch + if container.ImageManifest != nil { + imageManifestBytes, err := content.ReadBlob(ctx, cs, *container.ImageManifest) + if err != nil { + return "", err + } - var manifest ocispec.Manifest - if err := json.Unmarshal(imageManifestBytes, &manifest); err != nil { - return "", err - } + if err := json.Unmarshal(imageManifestBytes, &parentManifest); err != nil { + return "", err + } - imageConfigBytes, err := content.ReadBlob(ctx, cs, manifest.Config) - if err != nil { - return "", err - } - var ociimage ocispec.Image - if err := json.Unmarshal(imageConfigBytes, &ociimage); err != nil { - return "", err + imageConfigBytes, err := content.ReadBlob(ctx, cs, parentManifest.Config) + if err != nil { + return "", err + } + if err := json.Unmarshal(imageConfigBytes, &parentImage); err != nil { + return "", err + } } var ( @@ -78,15 +78,19 @@ func (i *ImageService) CommitImage(ctx context.Context, cc backend.CommitConfig) if err != nil { return "", fmt.Errorf("failed to export layer: %w", err) } + imageConfig := generateCommitImageConfig(parentImage, diffID, cc) + + layers := parentManifest.Layers + if diffLayerDesc != nil { + rootfsID := identity.ChainID(imageConfig.RootFS.DiffIDs).String() - imageConfig := generateCommitImageConfig(ociimage, diffID, cc) + if err := applyDiffLayer(ctx, rootfsID, parentImage, sn, differ, *diffLayerDesc); err != nil { + return "", fmt.Errorf("failed to apply diff: %w", err) + } - rootfsID := identity.ChainID(imageConfig.RootFS.DiffIDs).String() - if err := applyDiffLayer(ctx, rootfsID, ociimage, sn, differ, diffLayerDesc); err != nil { - return "", fmt.Errorf("failed to apply diff: %w", err) + layers = append(layers, *diffLayerDesc) } - layers := append(manifest.Layers, diffLayerDesc) commitManifestDesc, err := writeContentsForImage(ctx, container.Driver, cs, imageConfig, layers) if err != nil { return "", err @@ -130,6 +134,12 @@ func generateCommitImageConfig(baseConfig ocispec.Image, diffID digest.Digest, o logrus.Warnf("assuming os=%q", os) } logrus.Debugf("generateCommitImageConfig(): arch=%q, os=%q", arch, os) + + diffIds := baseConfig.RootFS.DiffIDs + if diffID != "" { + diffIds = append(diffIds, diffID) + } + return ocispec.Image{ Platform: ocispec.Platform{ Architecture: arch, @@ -140,7 +150,7 @@ func generateCommitImageConfig(baseConfig ocispec.Image, diffID digest.Digest, o Config: containerConfigToOciImageConfig(opts.Config), RootFS: ocispec.RootFS{ Type: "layers", - DiffIDs: append(baseConfig.RootFS.DiffIDs, diffID), + DiffIDs: diffIds, }, History: append(baseConfig.History, ocispec.History{ Created: &createdTime, @@ -217,28 +227,43 @@ func writeContentsForImage(ctx context.Context, snName string, cs content.Store, } // createDiff creates a layer diff into containerd's content store. -func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs content.Store, comparer diff.Comparer) (ocispec.Descriptor, digest.Digest, error) { +// If the diff is empty it returns nil empty digest and no error. +func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs content.Store, comparer diff.Comparer) (*ocispec.Descriptor, digest.Digest, error) { newDesc, err := rootfs.CreateDiff(ctx, name, sn, comparer) if err != nil { - return ocispec.Descriptor{}, "", err + return nil, "", err + } + + ra, err := cs.ReaderAt(ctx, newDesc) + if err != nil { + return nil, "", fmt.Errorf("failed to read diff archive: %w", err) + } + defer ra.Close() + + empty, err := archive.IsEmpty(content.NewReader(ra)) + if err != nil { + return nil, "", fmt.Errorf("failed to check if archive is empty: %w", err) + } + if empty { + return nil, "", nil } info, err := cs.Info(ctx, newDesc.Digest) if err != nil { - return ocispec.Descriptor{}, "", err + return nil, "", fmt.Errorf("failed to get content info: %w", err) } diffIDStr, ok := info.Labels["containerd.io/uncompressed"] if !ok { - return ocispec.Descriptor{}, "", fmt.Errorf("invalid differ response with no diffID") + return nil, "", fmt.Errorf("invalid differ response with no diffID") } diffID, err := digest.Parse(diffIDStr) if err != nil { - return ocispec.Descriptor{}, "", err + return nil, "", err } - return ocispec.Descriptor{ + return &ocispec.Descriptor{ MediaType: ocispec.MediaTypeImageLayerGzip, Digest: newDesc.Digest, Size: info.Size, @@ -254,7 +279,7 @@ func applyDiffLayer(ctx context.Context, name string, baseImg ocispec.Image, sn mount, err := sn.Prepare(ctx, key, parent) if err != nil { - return err + return fmt.Errorf("failed to prepare snapshot: %w", err) } defer func() { diff --git a/pkg/archive/diff.go b/pkg/archive/diff.go index c8c7be74797e8..1a2fb971f979c 100644 --- a/pkg/archive/diff.go +++ b/pkg/archive/diff.go @@ -223,6 +223,25 @@ func ApplyUncompressedLayer(dest string, layer io.Reader, options *TarOptions) ( return applyLayerHandler(dest, layer, options, false) } +// IsEmpty checks if the tar archive is empty (doesn't contain any entries). +func IsEmpty(rd io.Reader) (bool, error) { + decompRd, err := DecompressStream(rd) + if err != nil { + return true, fmt.Errorf("failed to decompress archive: %v", err) + } + defer decompRd.Close() + + tarReader := tar.NewReader(decompRd) + if _, err := tarReader.Next(); err != nil { + if err == io.EOF { + return true, nil + } + return false, fmt.Errorf("failed to read next archive header: %v", err) + } + + return false, nil +} + // do the bulk load of ApplyLayer, but allow for not calling DecompressStream func applyLayerHandler(dest string, layer io.Reader, options *TarOptions, decompress bool) (int64, error) { dest = filepath.Clean(dest) From 63422515ba0ef223ae78f5d9a7a09b08855f72cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 21 Aug 2023 16:50:49 +0200 Subject: [PATCH 234/293] c8d/run: Allow running container without image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows the legacy builder to apply changes to the `FROM scratch` layer. Signed-off-by: Paweł Gronowski (cherry picked from commit dfaff9598cf9207aaf2e275248d9cf8e8b9d3b54) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/image_snapshot.go | 57 +++++++++++++++-------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/daemon/containerd/image_snapshot.go b/daemon/containerd/image_snapshot.go index 962c2100dc223..21e36a4f7c75b 100644 --- a/daemon/containerd/image_snapshot.go +++ b/daemon/containerd/image_snapshot.go @@ -18,42 +18,45 @@ import ( // PrepareSnapshot prepares a snapshot from a parent image for a container func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *ocispec.Platform) error { - img, err := i.resolveImage(ctx, parentImage) - if err != nil { - return err - } + var parentSnapshot string + if parentImage != "" { + img, err := i.resolveImage(ctx, parentImage) + if err != nil { + return err + } - cs := i.client.ContentStore() + cs := i.client.ContentStore() - matcher := platforms.Default() - if platform != nil { - matcher = platforms.Only(*platform) - } + matcher := platforms.Default() + if platform != nil { + matcher = platforms.Only(*platform) + } - platformImg := containerd.NewImageWithPlatform(i.client, img, matcher) - unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter) - if err != nil { - return err - } + platformImg := containerd.NewImageWithPlatform(i.client, img, matcher) + unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter) + if err != nil { + return err + } - if !unpacked { - if err := platformImg.Unpack(ctx, i.snapshotter); err != nil { + if !unpacked { + if err := platformImg.Unpack(ctx, i.snapshotter); err != nil { + return err + } + } + + desc, err := containerdimages.Config(ctx, cs, img.Target, matcher) + if err != nil { return err } - } - desc, err := containerdimages.Config(ctx, cs, img.Target, matcher) - if err != nil { - return err - } + diffIDs, err := containerdimages.RootFS(ctx, cs, desc) + if err != nil { + return err + } - diffIDs, err := containerdimages.RootFS(ctx, cs, desc) - if err != nil { - return err + parentSnapshot = identity.ChainID(diffIDs).String() } - parent := identity.ChainID(diffIDs).String() - // Add a lease so that containerd doesn't garbage collect our snapshot ls := i.client.LeasesService() lease, err := ls.Create(ctx, leases.WithID(id)) @@ -69,7 +72,7 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentIma } s := i.client.SnapshotService(i.StorageDriver()) - _, err = s.Prepare(ctx, id, parent) + _, err = s.Prepare(ctx, id, parentSnapshot) return err } From 1d10e8633d5876517c689b788e39533370cb2c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 21 Aug 2023 16:52:32 +0200 Subject: [PATCH 235/293] daemon: Handle NotFound when deleting container lease MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the lease doesn't exit (for example when creating the container failed), just ignore the not found error. Signed-off-by: Paweł Gronowski (cherry picked from commit bedcc94de4cab9f3072a98f71806bc18e47cd072) Signed-off-by: Sebastiaan van Stijn --- daemon/delete.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/daemon/delete.go b/daemon/delete.go index 80cddd91a741d..bfd974e5b413c 100644 --- a/daemon/delete.go +++ b/daemon/delete.go @@ -8,6 +8,7 @@ import ( "strings" "time" + cerrdefs "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/leases" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" @@ -144,8 +145,10 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, config ty ID: container.ID, } if err := ls.Delete(context.Background(), lease, leases.SynchronousDelete); err != nil { - container.SetRemovalError(err) - return err + if !cerrdefs.IsNotFound(err) { + container.SetRemovalError(err) + return err + } } } } From ed2f5d1d85bfd976652b9879be20fcf8a33fbe76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 23 Aug 2023 13:53:46 +0200 Subject: [PATCH 236/293] c8d/builder: Don't drop fields from created image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous image created a new partially filled image. This caused child images to lose their parent's layers. Instead of creating a new object and trying to replace its fields, just clone the original passed image and change its ID to the manifest digest. Signed-off-by: Paweł Gronowski (cherry picked from commit 01214bafd211d1f8c74c1147c39338b27372ede5) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_builder.go | 5 +---- image/image.go | 9 +++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/daemon/containerd/image_builder.go b/daemon/containerd/image_builder.go index 05338da2dada5..fb864acd05855 100644 --- a/daemon/containerd/image_builder.go +++ b/daemon/containerd/image_builder.go @@ -505,9 +505,6 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st return nil, err } - newImage := dimage.NewImage(dimage.ID(createdImage.Target.Digest)) - newImage.V1Image = imgToCreate.V1Image - newImage.V1Image.ID = string(createdImage.Target.Digest) - newImage.History = imgToCreate.History + newImage := dimage.Clone(imgToCreate, dimage.ID(createdImage.Target.Digest)) return newImage, nil } diff --git a/image/image.go b/image/image.go index 856a845b17f98..d17601323ad0f 100644 --- a/image/image.go +++ b/image/image.go @@ -248,6 +248,15 @@ func NewChildImage(img *Image, child ChildConfig, os string) *Image { } } +// Clone clones an image and changes ID. +func Clone(base *Image, id ID) *Image { + img := *base + img.RootFS = img.RootFS.Clone() + img.V1Image.ID = id.String() + img.computedID = id + return &img +} + // History stores build commands that were used to create an image type History struct { // Created is the timestamp at which the image was created From 088cec8f0f592a01083d6fcf085e1a857860e6bd Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:30:15 +0200 Subject: [PATCH 237/293] hack: update link to GOPATH documentation This documentation moved to a different page, and the Go documentation moved to the https://go.dev/ domain. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 2aabd6447753bde5c844b2549cd2330cb1383bb6) Signed-off-by: Sebastiaan van Stijn --- hack/make.ps1 | 2 +- hack/make.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/make.ps1 b/hack/make.ps1 index 82a38f3cfce3c..5c9d3a951d34f 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -459,7 +459,7 @@ Try { if (-not $inContainer) { Verify-GoVersion } # Verify GOPATH is set - if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://golang.org/doc/code.html#GOPATH" } + if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://pkg.go.dev/cmd/go#hdr-GOPATH_environment_variable" } # Run autogen if building daemon. if ($Daemon) { diff --git a/hack/make.sh b/hack/make.sh index d8c7667cda4be..ec01bc1068cb5 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -79,7 +79,7 @@ if [ "$AUTO_GOPATH" ]; then fi if [ ! "$GOPATH" ]; then - echo >&2 'error: missing GOPATH; please see https://golang.org/doc/code.html#GOPATH' + echo >&2 'error: missing GOPATH; please see https://pkg.go.dev/cmd/go#hdr-GOPATH_environment_variable' echo >&2 ' alternatively, set AUTO_GOPATH=1' exit 1 fi From 377af4c9b4ee16f178a7899fdf953a99ad8daf06 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:45:50 +0200 Subject: [PATCH 238/293] Dockerfile: Windows: update Golang download domains to cut down redirects The `golang.org` domain moved to `go.dev`, and the download-URLs we were using resulted in 2 redirects; curl -sI https://golang.org/dl/go1.20.windows-amd64.zip | grep 'location' location: https://go.dev/dl/go1.20.windows-amd64.zip curl -sI https://go.dev/dl/go1.20.windows-amd64.zip | grep 'location' location: https://dl.google.com/go/go1.20.windows-amd64.zip curl -sI https://dl.google.com/go/go1.20.windows-amd64.zip HTTP/2 200 # ... This patch cuts it down to one redirects. I decided not to use the "final" (`dl.google.com`) URL, because that URL is not documented in the Golang docs, and visiting the domain itself (https://dl.google.com/) redirects to a marketing page for "Google Chrome". Trying the `/go/` path (https://dl.google.com/go/) also does not show a landing page that lists downloads, so I'm considering those URLs to be "unstable". Signed-off-by: Sebastiaan van Stijn (cherry picked from commit f6a5318f9411ae26bb6d7335b36505c83bbc21b9) Signed-off-by: Sebastiaan van Stijn --- Dockerfile.windows | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index d3783a9331bf7..2f20f52112413 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -224,7 +224,7 @@ RUN ` ` Write-Host INFO: Downloading go...; ` $dlGoVersion=$Env:GO_VERSION -replace '\.0$',''; ` - Download-File "https://golang.org/dl/go${dlGoVersion}.windows-amd64.zip" C:\go.zip; ` + Download-File "https://go.dev/dl/go${dlGoVersion}.windows-amd64.zip" C:\go.zip; ` ` Write-Host INFO: Downloading compiler 1 of 3...; ` Download-File https://raw.githubusercontent.com/moby/docker-tdmgcc/master/gcc.zip C:\gcc.zip; ` From de13951b9d0fb61595aeec48a14d534cb2fdad07 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:21:50 +0200 Subject: [PATCH 239/293] docs/api: update links to Go documentation Go documentation moved to the `go.dev` domain; curl -sI https://golang.org/doc/install/source#environment | grep 'location' location: https://go.dev/doc/install/source Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 4862d391442df10caca6f57104876deb754296d1) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.32.yaml | 4 ++-- docs/api/v1.33.yaml | 4 ++-- docs/api/v1.34.yaml | 4 ++-- docs/api/v1.35.yaml | 4 ++-- docs/api/v1.36.yaml | 4 ++-- docs/api/v1.37.yaml | 4 ++-- docs/api/v1.38.yaml | 4 ++-- docs/api/v1.39.yaml | 4 ++-- docs/api/v1.40.yaml | 4 ++-- docs/api/v1.41.yaml | 4 ++-- docs/api/v1.42.yaml | 4 ++-- docs/api/v1.43.yaml | 4 ++-- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/api/v1.32.yaml b/docs/api/v1.32.yaml index b843bc63406c4..ca9178c634f98 100644 --- a/docs/api/v1.32.yaml +++ b/docs/api/v1.32.yaml @@ -3632,7 +3632,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3640,7 +3640,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.33.yaml b/docs/api/v1.33.yaml index 3fa7eb43d5c58..fbf8476a13788 100644 --- a/docs/api/v1.33.yaml +++ b/docs/api/v1.33.yaml @@ -3637,7 +3637,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3645,7 +3645,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.34.yaml b/docs/api/v1.34.yaml index d90d4dbbf84d4..42cff6187d901 100644 --- a/docs/api/v1.34.yaml +++ b/docs/api/v1.34.yaml @@ -3666,7 +3666,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3674,7 +3674,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.35.yaml b/docs/api/v1.35.yaml index a3ed1494669fe..cb1c8d3709825 100644 --- a/docs/api/v1.35.yaml +++ b/docs/api/v1.35.yaml @@ -3648,7 +3648,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3656,7 +3656,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.36.yaml b/docs/api/v1.36.yaml index d5f4da5440eb6..a656e106bd6e0 100644 --- a/docs/api/v1.36.yaml +++ b/docs/api/v1.36.yaml @@ -3661,7 +3661,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3669,7 +3669,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.37.yaml b/docs/api/v1.37.yaml index b9290e1fea1dc..29a6640583f4e 100644 --- a/docs/api/v1.37.yaml +++ b/docs/api/v1.37.yaml @@ -3681,7 +3681,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3689,7 +3689,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.38.yaml b/docs/api/v1.38.yaml index af7b11817d86e..c1473afeee5c5 100644 --- a/docs/api/v1.38.yaml +++ b/docs/api/v1.38.yaml @@ -3735,7 +3735,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3743,7 +3743,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.39.yaml b/docs/api/v1.39.yaml index b7d7943af5b79..4102875ad8bc4 100644 --- a/docs/api/v1.39.yaml +++ b/docs/api/v1.39.yaml @@ -4719,7 +4719,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -4727,7 +4727,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.40.yaml b/docs/api/v1.40.yaml index 98df0853939e9..0d33d9908f111 100644 --- a/docs/api/v1.40.yaml +++ b/docs/api/v1.40.yaml @@ -4855,7 +4855,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -4863,7 +4863,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.41.yaml b/docs/api/v1.41.yaml index 13628c4348dd3..20cd26f5e5688 100644 --- a/docs/api/v1.41.yaml +++ b/docs/api/v1.41.yaml @@ -5006,7 +5006,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -5014,7 +5014,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.42.yaml b/docs/api/v1.42.yaml index ab1a740d30a14..a452c486b253c 100644 --- a/docs/api/v1.42.yaml +++ b/docs/api/v1.42.yaml @@ -5036,7 +5036,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -5044,7 +5044,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.43.yaml b/docs/api/v1.43.yaml index a616794c1e2f3..08cf7e52b674c 100644 --- a/docs/api/v1.43.yaml +++ b/docs/api/v1.43.yaml @@ -5068,7 +5068,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -5076,7 +5076,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: From 73f6053bb3183eb568a7eed6e08e071216e904bf Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:22:19 +0200 Subject: [PATCH 240/293] api: swagger: update link to Go documentation Go documentation moved to the `go.dev` domain; curl -sI https://golang.org/doc/install/source#environment | grep 'location' location: https://go.dev/doc/install/source Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 136e86bb5c08aaaf87fb8119f883133c077e3bcf) Signed-off-by: Sebastiaan van Stijn --- api/swagger.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index a820f996f94f6..7635b9f665ab2 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -5068,7 +5068,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -5076,7 +5076,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: From d2e9a19358a2e2dd4e0beca007ed9ba489b7ea76 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:28:05 +0200 Subject: [PATCH 241/293] CONTRIBUTING.md: update links to golang docs and blog - docs moved to https://go.dev/doc/ - blog moved to https://go.dev/blog/ Signed-off-by: Sebastiaan van Stijn (cherry picked from commit b18e170631998d9fdea893d6b3961aecc65cebae) Signed-off-by: Sebastiaan van Stijn --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 531ac610be4b8..e3bf263a9813c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -422,6 +422,6 @@ The rules: guidelines. Since you've read all the rules, you now know that. If you are having trouble getting into the mood of idiomatic Go, we recommend -reading through [Effective Go](https://golang.org/doc/effective_go.html). The -[Go Blog](https://blog.golang.org) is also a great resource. Drinking the +reading through [Effective Go](https://go.dev/doc/effective_go). The +[Go Blog](https://go.dev/blog/) is also a great resource. Drinking the kool-aid is a lot easier than going thirsty. From 1d983e2e8acb4324a5a8df77033d35787e96efdc Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 02:19:35 +0200 Subject: [PATCH 242/293] update golangci-lint to v1.54.2 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit cd49f9affdad81360620019b6e8741ae73fd5590) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- daemon/logger/splunk/splunk.go | 2 +- hack/validate/golangci-lint.yml | 11 +++++------ testutil/environment/special_images.go | 12 ++++++++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e4b61fa5b031..9c3716868f036 100644 --- a/Dockerfile +++ b/Dockerfile @@ -228,7 +228,7 @@ FROM binary-dummy AS containerd-windows FROM containerd-${TARGETOS} AS containerd FROM base AS golangci_lint -ARG GOLANGCI_LINT_VERSION=v1.51.2 +ARG GOLANGCI_LINT_VERSION=v1.54.2 RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ GOBIN=/build/ GO111MODULE=on go install "github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}" \ diff --git a/daemon/logger/splunk/splunk.go b/daemon/logger/splunk/splunk.go index d194334dadc80..7b495d24c2d14 100644 --- a/daemon/logger/splunk/splunk.go +++ b/daemon/logger/splunk/splunk.go @@ -37,7 +37,7 @@ const ( splunkCANameKey = "splunk-caname" splunkInsecureSkipVerifyKey = "splunk-insecureskipverify" splunkFormatKey = "splunk-format" - splunkVerifyConnectionKey = "splunk-verify-connection" + splunkVerifyConnectionKey = "splunk-verify-connection" // #nosec G101 -- ignoring: Potential hardcoded credentials (gosec) splunkGzipCompressionKey = "splunk-gzip" splunkGzipCompressionLevelKey = "splunk-gzip-level" splunkIndexAcknowledgment = "splunk-index-acknowledgment" diff --git a/hack/validate/golangci-lint.yml b/hack/validate/golangci-lint.yml index 2832ad7e77352..89221ec2da895 100644 --- a/hack/validate/golangci-lint.yml +++ b/hack/validate/golangci-lint.yml @@ -40,12 +40,11 @@ linters-settings: govet: check-shadowing: false depguard: - list-type: blacklist - include-go-root: true - packages: - # The io/ioutil package has been deprecated. - # https://go.dev/doc/go1.16#ioutil - - io/ioutil + rules: + main: + deny: + - pkg: io/ioutil + desc: The io/ioutil package has been deprecated, see https://go.dev/doc/go1.16#ioutil revive: rules: # FIXME make sure all packages have a description. Currently, there's many packages without. diff --git a/testutil/environment/special_images.go b/testutil/environment/special_images.go index a832cd7c3c645..e50efc8743543 100644 --- a/testutil/environment/special_images.go +++ b/testutil/environment/special_images.go @@ -1,10 +1,14 @@ package environment -// Graph driver image store identifies images by the ID of their config. -const DanglingImageIdGraphDriver = "sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43" +// DanglingImageIdGraphDriver is the digest for dangling images used +// in tests when the graph driver is used. The graph driver image store +// identifies images by the ID of their config. +const DanglingImageIdGraphDriver = "sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43" // #nosec G101 -- ignoring: Potential hardcoded credentials (gosec) -// The containerd image store identifies images by the ID of their manifest/manifest list. -const DanglingImageIdSnapshotter = "sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456" +// DanglingImageIdSnapshotter is the digest for dangling images used in +// tests when the containerd image store is used. The container image +// store identifies images by the ID of their manifest/manifest list.. +const DanglingImageIdSnapshotter = "sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456" // #nosec G101 -- ignoring: Potential hardcoded credentials (gosec) func GetTestDanglingImageId(testEnv *Execution) string { if testEnv.UsingSnapshotter() { From 5d4cc0b5b576dd7123c65ba3b030d194507bd829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 17 Aug 2023 16:44:22 +0200 Subject: [PATCH 243/293] integration/liveRestore: Check volume content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that the content in the live-restored volume mounted in a new container is the same as the content in the old container. This checks if volume's _data directory doesn't get unmounted on startup. Signed-off-by: Paweł Gronowski (cherry picked from commit aef703fa1b85faffb6c13db8860b3deecbb9de2f) Signed-off-by: Sebastiaan van Stijn --- integration/daemon/daemon_test.go | 52 ++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index f4565c9616766..5ce70d8418e56 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -1,14 +1,17 @@ package daemon // import "github.com/docker/docker/integration/daemon" import ( + "bytes" "context" "fmt" + "io" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" + "strings" "syscall" "testing" @@ -16,7 +19,9 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/daemon/config" + "github.com/docker/docker/errdefs" "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/testutil/daemon" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -431,9 +436,28 @@ func testLiveRestoreVolumeReferences(t *testing.T) { Source: v.Name, Target: "/foo", } - cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("top")) + + const testContent = "hello" + cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("sh", "-c", "echo "+testContent+">>/foo/test.txt; sleep infinity")) defer c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) + // Wait until container creates a file in the volume. + poll.WaitOn(t, func(t poll.LogT) poll.Result { + stat, err := c.ContainerStatPath(ctx, cID, "/foo/test.txt") + if err != nil { + if errdefs.IsNotFound(err) { + return poll.Continue("file doesn't yet exist") + } + return poll.Error(err) + } + + if int(stat.Size) != len(testContent)+1 { + return poll.Error(fmt.Errorf("unexpected test file size: %d", stat.Size)) + } + + return poll.Success() + }) + d.Restart(t, "--live-restore", "--iptables=false") // Try to remove the volume @@ -441,6 +465,32 @@ func testLiveRestoreVolumeReferences(t *testing.T) { err = c.VolumeRemove(ctx, v.Name, false) assert.ErrorContains(t, err, "volume is in use") + t.Run("volume still mounted", func(t *testing.T) { + skip.If(t, testEnv.IsRootless(), "restarted rootless daemon has a new mount namespace and it won't have the previous mounts") + + // Check if a new container with the same volume has access to the previous content. + // This fails if the volume gets unmounted at startup. + cID2 := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("cat", "/foo/test.txt")) + defer c.ContainerRemove(ctx, cID2, types.ContainerRemoveOptions{Force: true}) + + poll.WaitOn(t, container.IsStopped(ctx, c, cID2)) + + inspect, err := c.ContainerInspect(ctx, cID2) + if assert.Check(t, err) { + assert.Check(t, is.Equal(inspect.State.ExitCode, 0), "volume doesn't have the same file") + } + + logs, err := c.ContainerLogs(ctx, cID2, types.ContainerLogsOptions{ShowStdout: true}) + assert.NilError(t, err) + defer logs.Close() + + var stdoutBuf bytes.Buffer + _, err = stdcopy.StdCopy(&stdoutBuf, io.Discard, logs) + assert.NilError(t, err) + + assert.Check(t, is.Equal(strings.TrimSpace(stdoutBuf.String()), testContent)) + }) + // Remove that container which should free the references in the volume err = c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) assert.NilError(t, err) From c35376c4558a4d69c872e2a6cba72455f3dcb494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 17 Aug 2023 16:44:28 +0200 Subject: [PATCH 244/293] volume/local: Don't unmount, restore mounted status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On startup all local volumes were unmounted as a cleanup mechanism for the non-clean exit of the last engine process. This caused live-restored volumes that used special volume opt mount flags to be broken. While the refcount was restored, the _data directory was just unmounted, so all new containers mounting this volume would just have the access to the empty _data directory instead of the real volume. With this patch, the mountpoint isn't unmounted. Instead, if the volume is already mounted, just mark it as mounted, so the next time Mount is called only the ref count is incremented, but no second attempt to mount it is performed. Signed-off-by: Paweł Gronowski (cherry picked from commit 26894844026d20938ec84ea22448172d3af9b652) Signed-off-by: Sebastiaan van Stijn --- volume/local/local.go | 17 +++++++++++++---- volume/local/local_unix.go | 27 +++++++++++++++++++++++---- volume/local/local_windows.go | 5 +++++ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/volume/local/local.go b/volume/local/local.go index 1417941cca3a7..b4f3a3669a842 100644 --- a/volume/local/local.go +++ b/volume/local/local.go @@ -83,13 +83,18 @@ func New(scope string, rootIdentity idtools.Identity) (*Root, error) { quotaCtl: r.quotaCtl, } - // unmount anything that may still be mounted (for example, from an - // unclean shutdown). This is a no-op on windows - unmount(v.path) - if err := v.loadOpts(); err != nil { return nil, err } + + if err := v.restoreIfMounted(); err != nil { + log.G(context.TODO()).WithFields(log.Fields{ + "volume": v.name, + "path": v.path, + "error": err, + }).Warn("restoreIfMounted failed") + } + r.volumes[name] = v } @@ -338,6 +343,10 @@ func (v *localVolume) Unmount(id string) error { return nil } + if !v.active.mounted { + return nil + } + logger.Debug("Unmounting volume") return v.unmount() } diff --git a/volume/local/local_unix.go b/volume/local/local_unix.go index 2db5f8ef09205..0b534653b99c0 100644 --- a/volume/local/local_unix.go +++ b/volume/local/local_unix.go @@ -99,10 +99,6 @@ func (v *localVolume) setOpts(opts map[string]string) error { return v.saveOpts() } -func unmount(path string) { - _ = mount.Unmount(path) -} - func (v *localVolume) needsMount() bool { if v.opts == nil { return false @@ -163,6 +159,29 @@ func (v *localVolume) unmount() error { return nil } +// restoreIfMounted restores the mounted status if the _data directory is already mounted. +func (v *localVolume) restoreIfMounted() error { + if v.needsMount() { + // Check if the _data directory is already mounted. + mounted, err := mountinfo.Mounted(v.path) + if err != nil { + return fmt.Errorf("failed to determine if volume _data path is already mounted: %w", err) + } + + if mounted { + // Mark volume as mounted, but don't increment active count. If + // any container needs this, the refcount will be incremented + // by the live-restore (if enabled). + // In other case, refcount will be zero but the volume will + // already be considered as mounted when Mount is called, and + // only the refcount will be incremented. + v.active.mounted = true + } + } + + return nil +} + func (v *localVolume) CreatedAt() (time.Time, error) { fileInfo, err := os.Stat(v.rootPath) if err != nil { diff --git a/volume/local/local_windows.go b/volume/local/local_windows.go index 43b89b3cb15d5..11723b02f3956 100644 --- a/volume/local/local_windows.go +++ b/volume/local/local_windows.go @@ -43,6 +43,11 @@ func (v *localVolume) postMount() error { return nil } +// restoreIfMounted is a no-op on Windows (because mounts are not supported). +func (v *localVolume) restoreIfMounted() error { + return nil +} + func (v *localVolume) CreatedAt() (time.Time, error) { fileInfo, err := os.Stat(v.rootPath) if err != nil { From 8216da20afc8fd056a179e158174d70fc37f57fd Mon Sep 17 00:00:00 2001 From: Jean-Michel Rouet Date: Fri, 4 Nov 2022 11:27:25 +0100 Subject: [PATCH 245/293] more robust dockerd-rootless-setuptools.sh Fixing case where username may contain a backslash. This case can happen for winbind/samba active directory domain users. Signed-off-by: Jean-Michel Rouet Use more meaningful variable name Signed-off-by: Jean-Michel Rouet Update contrib/dockerd-rootless-setuptool.sh Co-authored-by: Akihiro Suda Signed-off-by: Jean-Michel Rouet Use more meaningful variable name Signed-off-by: Jean-Michel Rouet Update contrib/dockerd-rootless-setuptool.sh Co-authored-by: Akihiro Suda Signed-off-by: Jean-Michel Rouet (cherry picked from commit 2f0ba0a7e51756c9475d8b2379f32e4074e39afc) Signed-off-by: Ameya Gawde --- contrib/dockerd-rootless-setuptool.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/contrib/dockerd-rootless-setuptool.sh b/contrib/dockerd-rootless-setuptool.sh index c82bbc84604af..0c517a810f655 100755 --- a/contrib/dockerd-rootless-setuptool.sh +++ b/contrib/dockerd-rootless-setuptool.sh @@ -37,6 +37,8 @@ BIN="" SYSTEMD="" CFG_DIR="" XDG_RUNTIME_DIR_CREATED="" +USERNAME="" +USERNAME_ESCAPED="" # run checks and also initialize global vars init() { @@ -78,6 +80,11 @@ init() { exit 1 fi + # Set USERNAME from `id -un` and potentially protect backslash + # for windbind/samba domain users + USERNAME=$(id -un) + USERNAME_ESCAPED=$(echo $USERNAME | sed 's/\\/\\\\/g') + # set CFG_DIR CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}" @@ -222,21 +229,21 @@ init() { fi # instructions: validate subuid/subgid files for current user - if ! grep -q "^$(id -un):\|^$(id -u):" /etc/subuid 2> /dev/null; then + if ! grep -q "^$USERNAME_ESCAPED:\|^$(id -u):" /etc/subuid 2> /dev/null; then instructions=$( cat <<- EOI ${instructions} - # Add subuid entry for $(id -un) - echo "$(id -un):100000:65536" >> /etc/subuid + # Add subuid entry for ${USERNAME} + echo "${USERNAME}:100000:65536" >> /etc/subuid EOI ) fi - if ! grep -q "^$(id -un):\|^$(id -u):" /etc/subgid 2> /dev/null; then + if ! grep -q "^$USERNAME_ESCAPED:\|^$(id -u):" /etc/subgid 2> /dev/null; then instructions=$( cat <<- EOI ${instructions} - # Add subgid entry for $(id -un) - echo "$(id -un):100000:65536" >> /etc/subgid + # Add subgid entry for ${USERNAME} + echo "${USERNAME}:100000:65536" >> /etc/subgid EOI ) fi @@ -340,7 +347,7 @@ install_systemd() { ) INFO "Installed ${SYSTEMD_UNIT} successfully." INFO "To control ${SYSTEMD_UNIT}, run: \`systemctl --user (start|stop|restart) ${SYSTEMD_UNIT}\`" - INFO "To run ${SYSTEMD_UNIT} on system startup, run: \`sudo loginctl enable-linger $(id -un)\`" + INFO "To run ${SYSTEMD_UNIT} on system startup, run: \`sudo loginctl enable-linger ${USERNAME}\`" echo } From e2ab5f72eb77212084afc2aa930ccf1ac93d7290 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 8 Sep 2023 16:53:06 +0000 Subject: [PATCH 246/293] 24.0: Update buildkit to fix source policy order This brings in moby/buildkit#4215 which fixes a major issue with source policies in buildkit. Signed-off-by: Brian Goff --- builder/builder-next/worker/worker.go | 2 +- vendor.mod | 2 +- vendor.sum | 4 ++-- .../moby/buildkit/solver/llbsolver/solver.go | 16 +++++----------- vendor/modules.txt | 2 +- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index d91573c93f140..dad7331aeb198 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -50,7 +50,7 @@ import ( ) func init() { - version.Version = "v0.11.6+616c3f613b54" + version.Version = "v0.11.7+d3e6c1360f6e" } const labelCreatedAt = "buildkit/createdat" diff --git a/vendor.mod b/vendor.mod index 7cda78ef2bd90..38ef1bceb5c69 100644 --- a/vendor.mod +++ b/vendor.mod @@ -56,7 +56,7 @@ require ( github.com/klauspost/compress v1.16.3 github.com/miekg/dns v1.1.43 github.com/mistifyio/go-zfs/v3 v3.0.1 - github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go + github.com/moby/buildkit v0.11.7-0.20230908085316-d3e6c1360f6e // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go github.com/moby/ipvs v1.1.0 github.com/moby/locker v1.0.1 github.com/moby/patternmatcher v0.5.0 diff --git a/vendor.sum b/vendor.sum index 04af7872226f0..af7288f89a786 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1043,8 +1043,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ= -github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 h1:LSh03Csyx/zQq8MreC9MYMQE/+5EkohwZMvXSS6kMZo= -github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54/go.mod h1:bMQDryngJKGvJ/ZuRFhrejurbvYSv3NkGCheQ59X4AM= +github.com/moby/buildkit v0.11.7-0.20230908085316-d3e6c1360f6e h1:iDGoHMw0bMy+AVD59fVDM+jH6rivbFKCVF4iZ+Uw1Rc= +github.com/moby/buildkit v0.11.7-0.20230908085316-d3e6c1360f6e/go.mod h1:bMQDryngJKGvJ/ZuRFhrejurbvYSv3NkGCheQ59X4AM= github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go index d65a9e6490c7c..94d25ce5b7b28 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go @@ -977,27 +977,21 @@ func loadEntitlements(b solver.Builder) (entitlements.Set, error) { } func loadSourcePolicy(b solver.Builder) (*spb.Policy, error) { - set := make(map[spb.Rule]struct{}, 0) + var srcPol spb.Policy err := b.EachValue(context.TODO(), keySourcePolicy, func(v interface{}) error { x, ok := v.(spb.Policy) if !ok { return errors.Errorf("invalid source policy %T", v) } for _, f := range x.Rules { - set[*f] = struct{}{} + r := *f + srcPol.Rules = append(srcPol.Rules, &r) } + srcPol.Version = x.Version return nil }) if err != nil { return nil, err } - var srcPol *spb.Policy - if len(set) > 0 { - srcPol = &spb.Policy{} - for k := range set { - k := k - srcPol.Rules = append(srcPol.Rules, &k) - } - } - return srcPol, nil + return &srcPol, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index c3df1de562494..6de240d23534c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -577,7 +577,7 @@ github.com/mistifyio/go-zfs/v3 # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 -# github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 +# github.com/moby/buildkit v0.11.7-0.20230908085316-d3e6c1360f6e ## explicit; go 1.18 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types From f014c349a0fd00df3bdca57aed124a93d2f8d2de Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 11 Sep 2023 15:47:00 +0200 Subject: [PATCH 247/293] update to go1.20.8 go1.20.8 (released 2023-09-06) includes two security fixes to the html/template package, as well as bug fixes to the compiler, the go command, the runtime, and the crypto/tls, go/types, net/http, and path/filepath packages. See the Go 1.20.8 milestone on our issue tracker for details: https://github.com/golang/go/issues?q=milestone%3AGo1.20.8+label%3ACherryPickApproved full diff: https://github.com/golang/go/compare/go1.20.7...go1.20.8 From the security mailing: [security] Go 1.21.1 and Go 1.20.8 are released Hello gophers, We have just released Go versions 1.21.1 and 1.20.8, minor point releases. These minor releases include 4 security fixes following the security policy: - cmd/go: go.mod toolchain directive allows arbitrary execution The go.mod toolchain directive, introduced in Go 1.21, could be leveraged to execute scripts and binaries relative to the root of the module when the "go" command was executed within the module. This applies to modules downloaded using the "go" command from the module proxy, as well as modules downloaded directly using VCS software. Thanks to Juho Nurminen of Mattermost for reporting this issue. This is CVE-2023-39320 and Go issue https://go.dev/issue/62198. - html/template: improper handling of HTML-like comments within script contexts The html/template package did not properly handle HMTL-like "" comment tokens, nor hashbang "#!" comment tokens, in