[pull] main from containerd:main

cmd/ctr/commands/images/build_erofs_cache.go

@@ -44,7 +44,7 @@ var buildErofsCacheCommand = &cli.Command{
 	Description: `Convert each layer of an already-pulled image into a directory of
 diffID-keyed erofs blobs (<cache_dir>/<algorithm>/<xx>/<hex>.erofs, where <xx> is
 the first two characters of <hex>) for the erofs
-snapshotter's layer_content_cache. Layers are read from the content store; no
+snapshotter's layer_content_caches. Layers are read from the content store; no
 converted image is produced. The directory can then be synced to the read-only
 location the fleet mounts. Requires mkfs.erofs.
 

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/containerd/btrfs/v2 v2.0.0
 	github.com/containerd/cgroups/v3 v3.1.3
 	github.com/containerd/console v1.0.5
-	github.com/containerd/containerd/api v1.11.0
+	github.com/containerd/containerd/api v1.12.0-beta.0
 	github.com/containerd/continuity v0.5.0
 	github.com/containerd/errdefs v1.0.0
 	github.com/containerd/errdefs/pkg v0.3.0
@@ -164,5 +164,3 @@ require (
 	sigs.k8s.io/yaml v1.6.0 // indirect
 	tags.cncf.io/container-device-interface/specs-go v1.1.0 // indirect
 )
-
-replace github.com/containerd/containerd/api => ./api

go.sum

@@ -49,6 +49,8 @@ github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6
 github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw=
 github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc=
 github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
+github.com/containerd/containerd/api v1.12.0-beta.0 h1:0eGYh95iM9nn8Fygpx+96LvYIuc5xOiHvwo5hrsvTlI=
+github.com/containerd/containerd/api v1.12.0-beta.0/go.mod h1:/tQDq0fxPDGz9vrSpfhmFMfoR7s/uzvYMCY/qKygl9Y=
 github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
 github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=

plugins/snapshots/erofs/erofs.go

@@ -52,12 +52,13 @@ type SnapshotterConfig struct {
 	remapIDs    bool
 	// dmverityMode controls dm-verity behavior: "auto" (use if .dmverity exists), "on" (require .dmverity), "off" (disable)
 	dmverityMode string
-	// layerContentCache is a directory of pre-converted, diffID-keyed erofs
-	// layer blobs. When set and an unpacked layer's blob is present, the
-	// snapshotter stages the blob (as a symlink) into the active snapshot,
-	// skipping the download and tar->erofs conversion. Only parentless Prepares
-	// can be served. Empty disables the feature.
-	layerContentCache string
+	// layerContentCaches lists directories of pre-converted, diffID-keyed erofs
+	// layer blobs. Each is checked one by one; the first hit is staged into the
+	// snapshot (symlinked) instead of downloading and converting the layer. A
+	// directory that doesn't exist is treated as a cache miss. Layers missing
+	// from all of them are converted normally. Only parentless Prepares can be
+	// served.
+	layerContentCaches []string
 }
 
 // Opt is an option to configure the erofs snapshotter
@@ -105,13 +106,13 @@ func WithRemapIDs() Opt {
 	}
 }
 
-// WithLayerContentCache configures a read-only directory of pre-converted,
+// WithLayerContentCaches configures read-only directories of pre-converted,
 // diffID-keyed erofs layer blobs that the snapshotter sources layers from on
-// pull instead of downloading and converting them. See the layerContentCache
+// pull instead of downloading and converting them. See the layerContentCaches
 // field for details.
-func WithLayerContentCache(path string) Opt {
+func WithLayerContentCaches(paths ...string) Opt {
 	return func(config *SnapshotterConfig) {
-		config.layerContentCache = path
+		config.layerContentCaches = paths
 	}
 }
 
@@ -122,16 +123,16 @@ type MetaStore interface {
 }
 
 type snapshotter struct {
-	root              string
-	ms                MetaStore
-	ovlOptions        []string
-	enableFsverity    bool
-	setImmutable      bool
-	defaultWritable   int64
-	blockMode         bool
-	remapIDs          bool
-	dmverityMode      string
-	layerContentCache string
+	root               string
+	ms                 MetaStore
+	ovlOptions         []string
+	enableFsverity     bool
+	setImmutable       bool
+	defaultWritable    int64
+	blockMode          bool
+	remapIDs           bool
+	dmverityMode       string
+	layerContentCaches []string
 }
 
 // NewSnapshotter returns a Snapshotter which uses EROFS+OverlayFS. The layers
@@ -177,13 +178,25 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
 	// so fsverity and IMMUTABLE_FL can't be applied without mutating that blob out
 	// from under other snapshots. Reject them explicitly instead of silently
 	// skipping; dm-verity is the cache's integrity mechanism.
-	if config.layerContentCache != "" {
+	if len(config.layerContentCaches) > 0 {
 		if config.enableFsverity {
-			return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_cache; use dm-verity for cache integrity")
+			return nil, fmt.Errorf("enable_fsverity is incompatible with layer_content_caches; use dm-verity for cache integrity")
 		}
 		if config.setImmutable {
-			return nil, fmt.Errorf("set_immutable is incompatible with layer_content_cache")
+			return nil, fmt.Errorf("set_immutable is incompatible with layer_content_caches")
 		}
+
+		// A cache dir is symlinked into snapshots, so a relative one would resolve
+		// against the snapshot dir and dangle. The check is only lexical: dirs are
+		// not required to exist, as a missing one just yields a cache miss and may
+		// well be provisioned after startup.
+		for _, dir := range config.layerContentCaches {
+			if !filepath.IsAbs(dir) {
+				return nil, fmt.Errorf("layer_content_caches %q must be an absolute path", dir)
+			}
+		}
+
+		log.L.WithField("dirs", config.layerContentCaches).Info("erofs layer content cache enabled")
 	}
 
 	// Check fsverity support if enabled
@@ -202,24 +215,6 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
 		return nil, fmt.Errorf("setting IMMUTABLE_FL is only supported on Linux")
 	}
 
-	// Resolve the cache dir to an absolute path so materialized layer blobs are
-	// absolute symlinks, independent of the process working directory, and verify
-	// it exists and is a directory so misconfiguration fails fast at startup.
-	if config.layerContentCache != "" {
-		abs, err := filepath.Abs(config.layerContentCache)
-		if err != nil {
-			return nil, fmt.Errorf("failed to resolve layer_content_cache path %q: %w", config.layerContentCache, err)
-		}
-		fi, err := os.Stat(abs)
-		if err != nil {
-			return nil, fmt.Errorf("failed to access layer_content_cache %q: %w", abs, err)
-		}
-		if !fi.IsDir() {
-			return nil, fmt.Errorf("layer_content_cache %q is not a directory", abs)
-		}
-		config.layerContentCache = abs
-	}
-
 	ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db"))
 	if err != nil {
 		return nil, err
@@ -230,16 +225,16 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
 	}
 
 	return &snapshotter{
-		root:              root,
-		ms:                ms,
-		ovlOptions:        config.ovlOptions,
-		enableFsverity:    config.enableFsverity,
-		setImmutable:      config.setImmutable,
-		defaultWritable:   config.defaultSize,
-		blockMode:         config.defaultSize > 0,
-		remapIDs:          config.remapIDs,
-		dmverityMode:      config.dmverityMode,
-		layerContentCache: config.layerContentCache,
+		root:               root,
+		ms:                 ms,
+		ovlOptions:         config.ovlOptions,
+		enableFsverity:     config.enableFsverity,
+		setImmutable:       config.setImmutable,
+		defaultWritable:    config.defaultSize,
+		blockMode:          config.defaultSize > 0,
+		remapIDs:           config.remapIDs,
+		dmverityMode:       config.dmverityMode,
+		layerContentCaches: config.layerContentCaches,
 	}, nil
 }
 
@@ -700,21 +695,17 @@ func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s
 	return s.createSnapshot(ctx, snapshots.KindActive, key, parent, opts)
 }
 
-// cacheBlobPath returns the expected path of the cached erofs blob for a diffID.
-func (s *snapshotter) cacheBlobPath(diffID digest.Digest) string {
-	return erofsutils.CacheBlobPath(s.layerContentCache, diffID)
-}
-
 // lookupCache returns the absolute path of the cached erofs blob that can serve
-// the layer being prepared, or "" on a miss. It gates on: the cache being
-// configured, the Prepare being an image-layer extraction (carries the
-// snapshot.ref and diff-id labels), and the diffID blob being present. Misses
-// (cache disabled, non-extraction Prepare, missing entries, unreadable cache dirs
-// such as a FUSE mount being down, malformed labels) all return "" so pulls keep
-// working. Any dm-verity sidecar is derived from the blob path (via
-// dmverity.MetadataPath) when the blob is materialized (prepareDirectory).
+// the layer being prepared, or "" on a miss. It gates on: at least one cache
+// being configured, the Prepare being an image-layer extraction (carries the
+// snapshot.ref and diff-id labels), and the diffID blob being present. Caches
+// are checked one by one and the first hit wins. Every miss (cache disabled,
+// non-extraction Prepare, absent or unreadable cache dir, missing entry,
+// malformed labels) returns "" so pulls keep working. Any dm-verity sidecar is
+// derived from the blob path (via dmverity.MetadataPath) when the blob is
+// staged (prepareDirectory).
 func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) string {
-	if s.layerContentCache == "" {
+	if len(s.layerContentCaches) == 0 {
 		return ""
 	}
 
@@ -737,18 +728,24 @@ func (s *snapshotter) lookupCache(ctx context.Context, opts ...snapshots.Opt) st
 		return ""
 	}
 
-	blob := s.cacheBlobPath(diffID)
-	if _, err := os.Stat(blob); err != nil {
-		if os.IsNotExist(err) {
-			log.G(ctx).WithField("blob", blob).Trace("erofs layer cache miss")
-		} else {
-			log.G(ctx).WithError(err).WithField("blob", blob).
-				Warn("erofs layer cache: failed to stat cache blob, treating as cache miss")
+	for _, dir := range s.layerContentCaches {
+		blob := erofsutils.CacheBlobPath(dir, diffID)
+		if _, err := os.Stat(blob); err != nil {
+			if !os.IsNotExist(err) {
+				// An unreadable cache (a down FUSE mount, a permission change since
+				// startup) shouldn't fail the pull or mask a hit in a later cache.
+				log.G(ctx).WithError(err).WithField("blob", blob).
+					Warn("erofs layer cache: failed to stat cache blob, skipping this cache")
+			}
+			continue
 		}
-		return ""
+		// Absolute, since the configured dirs are validated as such: the hit is
+		// symlinked into the snapshot dir, where a relative target would dangle.
+		return blob
 	}
 
-	return blob
+	log.G(ctx).WithField("diffID", diffID.String()).Trace("erofs layer cache miss")
+	return ""
 }
 
 func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {

plugins/snapshots/erofs/erofs_linux_test.go

@@ -990,7 +990,7 @@ func TestCacheHit(t *testing.T) {
 	diffID := digest.Digest(cacheTestDiffID)
 	blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob"))
 
-	s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+	s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
 
 	target := cacheTestChainID
 	key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1031,7 +1031,7 @@ func TestCacheSidecar(t *testing.T) {
 	require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644))
 
 	// dmverity_mode defaults to "auto": use the sidecar if present.
-	s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+	s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
 
 	target := cacheTestChainID
 	key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1078,15 +1078,15 @@ func TestCacheMiss(t *testing.T) {
 	})
 
 	t.Run("blob absent", func(t *testing.T) {
-		s := newCacheSnapshotter(t, WithLayerContentCache(t.TempDir()))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(t.TempDir()))
 		mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID))
 		assertFellThrough(t, s, mounts, err, false)
 	})
 
 	t.Run("no extraction labels", func(t *testing.T) {
 		cacheDir := t.TempDir()
 		writeCacheBlob(t, cacheDir, diffID, []byte("blob"))
-		s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
 		// A container-rootfs Prepare carries no snapshot.ref/diff-id labels.
 		mounts, err := s.Prepare(ctx, "container-rootfs", "")
 		require.NoError(t, err)
@@ -1096,7 +1096,7 @@ func TestCacheMiss(t *testing.T) {
 	t.Run("view is never short-circuited", func(t *testing.T) {
 		cacheDir := t.TempDir()
 		writeCacheBlob(t, cacheDir, diffID, []byte("blob"))
-		s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
 		// Even with matching labels and a cached blob, a View must not commit.
 		// It's read-only, but via the KindView roFlag in mounts(), not the cache.
 		mounts, err := s.View(ctx, "view-1", "", extractionOpt(target, diffID))
@@ -1122,7 +1122,7 @@ func TestCacheParentedPrepare(t *testing.T) {
 	writeCacheBlob(t, cacheDir, parentDiffID, []byte("fake parent blob"))
 	writeCacheBlob(t, cacheDir, childDiffID, []byte("fake child blob"))
 
-	s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+	s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
 
 	// The first layer has no parent, so it is served from the cache as usual.
 	require.NoError(t, s.Commit(ctx, parentChain, stageCacheHit(t, ctx, s, parentChain, parentDiffID)))
@@ -1152,7 +1152,7 @@ func TestCacheRemove(t *testing.T) {
 	sidecar := dmverity.MetadataPath(blob)
 	require.NoError(t, os.WriteFile(sidecar, []byte(testDmverityMetadata), 0644))
 
-	s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir))
+	s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir))
 
 	target := cacheTestChainID
 	key := stageCacheHit(t, ctx, s, target, diffID)
@@ -1189,7 +1189,7 @@ func TestCacheDmverity(t *testing.T) {
 		blob := writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob"))
 		require.NoError(t, os.WriteFile(dmverity.MetadataPath(blob), []byte(testDmverityMetadata), 0644))
 
-		s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on"))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir), WithDmverityMode("on"))
 		key := stageCacheHit(t, ctx, s, target, diffID)
 
 		_, err := os.Stat(dmverity.MetadataPath(s.layerBlobPath(snapshotID(t, ctx, s, key))))
@@ -1200,7 +1200,7 @@ func TestCacheDmverity(t *testing.T) {
 		cacheDir := t.TempDir()
 		writeCacheBlob(t, cacheDir, diffID, []byte("fake erofs blob")) // no sidecar
 
-		s := newCacheSnapshotter(t, WithLayerContentCache(cacheDir), WithDmverityMode("on"))
+		s := newCacheSnapshotter(t, WithLayerContentCaches(cacheDir), WithDmverityMode("on"))
 
 		// dmverity_mode=on requires a sidecar; a cache entry without one is a hard
 		// error rather than a silent fallback.
@@ -1211,3 +1211,71 @@ func TestCacheDmverity(t *testing.T) {
 		assert.Error(t, err, "no snapshot should be committed on failure")
 	})
 }
+
+// TestCacheMultipleDirs covers the layer_content_caches search path: directories
+// are searched in configured order, the first hit wins, later caches are reached
+// when earlier ones lack the blob, and a miss in every cache falls through to a
+// normal writable snapshot.
+func TestCacheMultipleDirs(t *testing.T) {
+	ctx := namespaces.WithNamespace(context.Background(), "test")
+	diffID := digest.Digest(cacheTestDiffID)
+	target := cacheTestChainID
+
+	// stagedBlob returns the cache blob the snapshot's layer.erofs symlink points
+	// at, i.e. which of the configured caches actually served the layer.
+	stagedBlob := func(t *testing.T, s *snapshotter, key string) string {
+		t.Helper()
+		dst, err := os.Readlink(s.layerBlobPath(snapshotID(t, ctx, s, key)))
+		require.NoError(t, err)
+		return dst
+	}
+
+	t.Run("first cache with the blob wins", func(t *testing.T) {
+		first, second := t.TempDir(), t.TempDir()
+		// Both caches hold the diffID; the earlier one must be the one used.
+		firstBlob := writeCacheBlob(t, first, diffID, []byte("from first"))
+		writeCacheBlob(t, second, diffID, []byte("from second"))
+
+		s := newCacheSnapshotter(t, WithLayerContentCaches(first, second))
+		key := stageCacheHit(t, ctx, s, target, diffID)
+		assert.Equal(t, firstBlob, stagedBlob(t, s, key))
+	})
+
+	t.Run("falls through to a later cache", func(t *testing.T) {
+		empty, populated := t.TempDir(), t.TempDir()
+		blob := writeCacheBlob(t, populated, diffID, []byte("from second"))
+
+		s := newCacheSnapshotter(t, WithLayerContentCaches(empty, populated))
+		key := stageCacheHit(t, ctx, s, target, diffID)
+		assert.Equal(t, blob, stagedBlob(t, s, key))
+	})
+
+	t.Run("miss in every cache falls back to a writable snapshot", func(t *testing.T) {
+		s := newCacheSnapshotter(t, WithLayerContentCaches(
+			filepath.Join(t.TempDir(), "missing"), t.TempDir(), t.TempDir()))
+
+		mounts, err := s.Prepare(ctx, "extract-1 "+target, "", extractionOpt(target, diffID))
+		require.NoError(t, err)
+		require.NotEmpty(t, mounts)
+		for _, m := range mounts {
+			assert.False(t, m.ReadOnly(), "a miss in every cache must return writable mounts")
+		}
+		_, err = s.Stat(ctx, target)
+		assert.Error(t, err, "target chainID must not be committed on a miss")
+	})
+}
+
+// TestCacheDirMustBeAbsolute covers the one thing NewSnapshotter checks about a
+// configured cache dir: it must be absolute, since a relative one would be
+// symlinked into the snapshot dir and dangle. The empty string is covered by the
+// same check, which otherwise resolves to the daemon's working directory.
+func TestCacheDirMustBeAbsolute(t *testing.T) {
+	requireErofs(t)
+
+	for _, dir := range []string{"relative-cache", "./cache", "", "a/b"} {
+		t.Run(fmt.Sprintf("%q", dir), func(t *testing.T) {
+			_, err := NewSnapshotter(t.TempDir(), WithLayerContentCaches(dir))
+			assert.ErrorContains(t, err, "must be an absolute path")
+		})
+	}
+}