engine: retry a volume mount while the device is not yet openable

sandboxd/engine/installca_test.go

@@ -78,6 +78,7 @@ type fakeSilkd struct {
 	execEnv    map[string]string
 	execCode   int32
 	execFailAt int
+	notReady   int
 	execHook   func([]string)
 	realpath   map[string]string
 	noRealpath bool
@@ -221,6 +222,11 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri
 	if f.execFailAt > 0 && len(f.execCalls) != f.execFailAt {
 		code, reply = 0, ""
 	}
+	stderr := ""
+	if argv[0] == "mount" && f.notReady > 0 {
+		f.notReady--
+		code, stderr = 32, "mount: /data: /dev/vdc is not a valid block device."
+	}
 	if code == 0 && reply == "" && f.execHook != nil {
 		f.execHook(argv)
 	}
@@ -234,6 +240,9 @@ func (f *fakeSilkd) handleExec(conn net.Conn, argv []string, env map[string]stri
 		return
 	}
 	_, _ = io.WriteString(conn, `{"type":"started","pid":1}`+"\n")
+	if stderr != "" {
+		writeFakeSilkdResponse(conn, &wire.Stderr{Data: []byte(stderr)})
+	}
 	if argv[0] == "realpath" && code == 0 {
 		writeFakeSilkdResponse(conn, &wire.Stdout{Data: []byte(cmp.Or(resolved, argv[len(argv)-1]) + "\n")})
 	}

sandboxd/engine/volume.go

@@ -144,10 +144,24 @@ func (e *Engine) mountDevice(ctx context.Context, vsockSocket, device, name, mou
 	if err := e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", mount); err != nil {
 		return fmt.Errorf("create volume mount point %s: %w", mount, err)
 	}
-	if err := e.silkdExec(ctx, vsockSocket, "mount", "-o", mountMode(rw), "--", device, mount); err != nil {
-		return fmt.Errorf("mount volume %s: %w", name, err)
+	ticker := time.NewTicker(volumePollInterval)
+	defer ticker.Stop()
+	for {
+		err := e.silkdExec(ctx, vsockSocket, "mount", "-o", mountMode(rw), "--", device, mount)
+		switch {
+		case err == nil:
+			return nil
+		case ctx.Err() != nil:
+			return fmt.Errorf("mount volume %s: %w: %w", name, ctx.Err(), err)
+		case !deviceNotReady(err):
+			return fmt.Errorf("mount volume %s: %w", name, err)
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("mount volume %s: %w: %w", name, ctx.Err(), err)
+		case <-ticker.C:
+		}
 	}
-	return nil
 }
 
 func (e *Engine) diskAttachCopy(ctx context.Context, vmName, sourceFlag, source string) (string, error) {
@@ -180,6 +194,11 @@ func (e *Engine) diskAttachArgs(vmName string, spec VolumeSpec) ([]string, error
 	return append(args, "--directio", directIO), nil
 }
 
+func deviceNotReady(err error) bool {
+	exit, ok := errors.AsType[*guestExitError](err)
+	return ok && strings.Contains(exit.output, "not a valid block device")
+}
+
 func isNotFound(err error) bool {
 	respErr, ok := errors.AsType[*wire.ErrorResp](err)
 	return ok && respErr.Kind == wire.KindNotFound

sandboxd/engine/volume_test.go

@@ -226,6 +226,38 @@ func TestMountVolumeWaitsForDelayedDevNode(t *testing.T) {
 	}
 }
 
+func TestMountVolumeRetriesUntilTheDeviceOpens(t *testing.T) {
+	path := sockPath(t)
+	fake := serveFakeSilkd(t, path)
+	configureVolumeDevices(fake)
+	fake.mu.Lock()
+	fake.notReady = 2
+	fake.mu.Unlock()
+	if err := New("cocoon", nil, nil, false, false, "").MountVolume(t.Context(), path, "imagenet", "/datasets/training", false); err != nil {
+		t.Fatalf("MountVolume: %v", err)
+	}
+	fake.mu.Lock()
+	defer fake.mu.Unlock()
+	if mounts := len(fake.execCalls) - 1; mounts != 3 {
+		t.Fatalf("mount attempts = %d, want 3: %v", mounts, fake.execCalls)
+	}
+}
+
+func TestMountVolumeNotReadyUntilDeadlineReportsTheTimeout(t *testing.T) {
+	path := sockPath(t)
+	fake := serveFakeSilkd(t, path)
+	configureVolumeDevices(fake)
+	fake.mu.Lock()
+	fake.notReady = 1 << 20
+	fake.mu.Unlock()
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	err := New("cocoon", nil, nil, false, false, "").MountVolume(ctx, path, "imagenet", "/datasets/training", false)
+	if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "mount volume imagenet") {
+		t.Fatalf("got %v, want the mount deadline", err)
+	}
+}
+
 func TestMountVolumeStopsAtFailedStage(t *testing.T) {
 	for _, tt := range []struct {
 		name     string