diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 61e481d8dd..5befc04193 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -32,7 +32,6 @@ jobs: - 'e2e/**' - 'hack/microsandbox/**' - 'hack/provider_update_ci/**' - - 'hack/run-managed-policy-e2e.sh' - '.github/workflows/pr-ci.yml' - '.github/workflows/docker-buildkit-windows.yml' - 'hack/licenses/**' diff --git a/hack/microsandbox/main.go b/hack/microsandbox/main.go index 59762e1486..4ab924fa9c 100644 --- a/hack/microsandbox/main.go +++ b/hack/microsandbox/main.go @@ -11,10 +11,12 @@ import ( ) const ( - installLinuxCommand = "install-linux" - runSuiteCommand = "run" - verifyReportCommand = "verify-report" - buildPerformanceToolsCommand = "build-performance-tools" + installLinuxCommand = "install-linux" + runSuiteCommand = "run" + verifyReportCommand = "verify-report" + buildPerformanceToolsCommand = "build-performance-tools" + managedPolicyE2ECommand = "managed-policy-e2e" + managedPolicyNamespaceCommand = "managed-policy-namespace" ) func main() { @@ -25,10 +27,13 @@ func runCLI(args []string, stdout, stderr io.Writer) int { if len(args) == 0 { _, _ = fmt.Fprintln( stderr, - "usage: devsy-ci-microsandbox ", + "usage: devsy-ci-microsandbox ", ) return 2 } + if code, ok := runManagedPolicyCommand(args, stderr); ok { + return code + } switch args[0] { case installLinuxCommand: return installLinuxCLI(args[1:], stdout, stderr) @@ -44,6 +49,19 @@ func runCLI(args []string, stdout, stderr io.Writer) int { } } +// runManagedPolicyCommand dispatches the managed-policy commands and reports +// whether args[0] named one of them. +func runManagedPolicyCommand(args []string, stderr io.Writer) (int, bool) { + switch args[0] { + case managedPolicyE2ECommand: + return managedPolicyE2ECLI(args[1:], stderr), true + case managedPolicyNamespaceCommand: + return managedPolicyNamespaceCLI(args[1:], stderr), true + default: + return 0, false + } +} + func installLinuxCLI(args []string, stdout, stderr io.Writer) int { if len(args) != 0 || runtime.GOOS != "linux" { _, _ = fmt.Fprintln(stderr, "install-linux requires Linux and takes no arguments") diff --git a/hack/microsandbox/managed_policy_linux.go b/hack/microsandbox/managed_policy_linux.go new file mode 100644 index 0000000000..f7d51081ed --- /dev/null +++ b/hack/microsandbox/managed_policy_linux.go @@ -0,0 +1,347 @@ +//go:build linux + +package main + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "syscall" +) + +const ( + managedPolicyMarker = "DEVSY_REQUIRE_MSB_MANAGED_POLICY_FIXTURE" + managedPolicyTestBinary = "./e2e.test" + managedPolicyDocument = "{\"version\":1,\"overrides\":{}}\n" + managedPolicyIDFile = "mount-namespace-id" +) + +// managedPolicyRuntime isolates privileged side effects so tests can run the invariants unprivileged. +type managedPolicyRuntime struct { + euid int + uid, gid uint32 + etcRoot string + mountNamespace func() (string, error) + mountOverlay func(target, options string) error + readMountInfo func() (string, error) + lookPath func(string) (string, error) + executable func() (string, error) + replace func(path string, argv, env []string) error +} + +// managedPolicyNamespace is the namespace state handed to the test command. +type managedPolicyNamespace struct { + root, parent, current string +} + +// productionManagedPolicyRuntime leaves uid and gid zero, so fixture and policy files are root-owned. +func productionManagedPolicyRuntime() managedPolicyRuntime { + return managedPolicyRuntime{ + euid: os.Geteuid(), + etcRoot: "/etc", + mountNamespace: func() (string, error) { return os.Readlink("/proc/self/ns/mnt") }, + mountOverlay: func(target, options string) error { + return syscall.Mount("overlay", target, "overlay", 0, options) + }, + readMountInfo: func() (string, error) { + data, err := os.ReadFile("/proc/self/mountinfo") + return string(data), err + }, + lookPath: exec.LookPath, + executable: os.Executable, + replace: syscall.Exec, + } +} + +// managedPolicyExit runs one stage under a private umask; a successful stage never returns. +func managedPolicyExit( + stderr io.Writer, + stage func(managedPolicyRuntime, []string) error, + args []string, +) int { + syscall.Umask(0o077) + if err := stage(productionManagedPolicyRuntime(), args); err != nil { + _, _ = fmt.Fprintln(stderr, err) + return 1 + } + return 0 +} + +func managedPolicyE2ECLI(args []string, stderr io.Writer) int { + return managedPolicyExit(stderr, runManagedPolicyE2E, args) +} + +func managedPolicyNamespaceCLI(args []string, stderr io.Writer) int { + return managedPolicyExit(stderr, runManagedPolicyNamespace, args) +} + +// runManagedPolicyE2E creates the fixture and re-execs this binary inside a private mount namespace. +func runManagedPolicyE2E(rt managedPolicyRuntime, args []string) error { + runnerTemp := os.Getenv("RUNNER_TEMP") + if err := checkManagedPolicyInputs(rt, runnerTemp, args); err != nil { + return err + } + root, err := createManagedPolicyFixture(runnerTemp, rt.uid, rt.gid) + if err != nil { + return err + } + parent, err := rt.mountNamespace() + if err != nil { + return fmt.Errorf("read parent mount namespace: %w", err) + } + self, err := rt.executable() + if err != nil { + return fmt.Errorf("resolve managed-policy wrapper: %w", err) + } + unshare, err := rt.lookPath("unshare") + if err != nil { + return fmt.Errorf("find unshare: %w", err) + } + argv := append([]string{ + "unshare", "--mount", "--propagation", "private", "--", self, + managedPolicyNamespaceCommand, root, filepath.Join(root, managedPolicyIDFile), parent, + }, args...) + if err := rt.replace(unshare, argv, os.Environ()); err != nil { + return fmt.Errorf("exec unshare: %w", err) + } + return nil +} + +func checkManagedPolicyInputs(rt managedPolicyRuntime, runnerTemp string, args []string) error { + info, statErr := os.Stat(runnerTemp) + switch { + case rt.euid != 0: + return errors.New("managed-policy fixture must run as root") + case os.Getenv(managedPolicyMarker) != "true": + return errors.New("managed-policy E2E wrapper requires the CI expectation marker") + case statErr != nil || !info.IsDir(): + return errors.New("RUNNER_TEMP must name an existing directory") + case len(args) == 0 || args[0] != managedPolicyTestBinary: + return errors.New("expected the already-built ./e2e.test binary as the first argument") + } + return nil +} + +// createManagedPolicyFixture creates /devsy-managed-policy.* with upper and work +// subdirectories. The CLI umask keeps every directory at 0700; ownership is then verified. +func createManagedPolicyFixture(runnerTemp string, uid, gid uint32) (string, error) { + root, err := os.MkdirTemp(filepath.Clean(runnerTemp), "devsy-managed-policy.") + if err != nil { + return "", fmt.Errorf("create managed-policy fixture: %w", err) + } + dirs := []string{root, filepath.Join(root, "upper"), filepath.Join(root, "work")} + for _, dir := range dirs[1:] { + if err := os.Mkdir(dir, 0o700); err != nil { + return "", fmt.Errorf("create managed-policy fixture directory: %w", err) + } + } + for _, dir := range dirs { + if err := verifyManagedPolicyDirectory(dir, uid, gid); err != nil { + return "", err + } + } + return root, nil +} + +// verifyManagedPolicyDirectory requires dir to be owned by uid:gid with mode 0700. +func verifyManagedPolicyDirectory(dir string, uid, gid uint32) error { + info, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("inspect managed-policy fixture directory %s: %w", dir, err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != uid || stat.Gid != gid || stat.Mode&0o7777 != 0o700 { + return fmt.Errorf( + "managed-policy fixture directory has unexpected ownership or mode: %s", + dir, + ) + } + return nil +} + +// runManagedPolicyNamespace runs inside the unshared namespace: it overlays /etc, installs the +// initial policy, and execs the test command. +func runManagedPolicyNamespace(rt managedPolicyRuntime, args []string) error { + ns, err := enterManagedPolicyNamespace(rt, args) + if err != nil { + return err + } + if err := mountManagedPolicyOverlay(rt, ns.root); err != nil { + return err + } + policy, err := installManagedPolicy(rt) + if err != nil { + return err + } + command := args[3:] + if err := rt.replace( + command[0], + command, + managedPolicyChildEnvironment(os.Environ(), ns, policy), + ); err != nil { + return fmt.Errorf("exec %s: %w", command[0], err) + } + return nil +} + +// enterManagedPolicyNamespace checks that the mount namespace changed and records its id. +func enterManagedPolicyNamespace( + rt managedPolicyRuntime, + args []string, +) (managedPolicyNamespace, error) { + if rt.euid != 0 { + return managedPolicyNamespace{}, errors.New("managed-policy namespace must run as root") + } + if len(args) < 4 { + return managedPolicyNamespace{}, errors.New( + "managed-policy-namespace requires ", + ) + } + ns := managedPolicyNamespace{root: args[0], parent: args[2]} + current, err := rt.mountNamespace() + if err != nil { + return ns, fmt.Errorf("read mount namespace: %w", err) + } + if current == ns.parent { + return ns, errors.New("mount namespace did not change") + } + ns.current = current + if err := os.WriteFile(args[1], []byte(current+"\n"), 0o600); err != nil { + return ns, fmt.Errorf("write mount namespace id: %w", err) + } + return ns, nil +} + +// mountManagedPolicyOverlay overlays etc before any managed-policy path is created, so no host +// file changes, and verifies the overlay uses this fixture's upper and work directories. +func mountManagedPolicyOverlay(rt managedPolicyRuntime, root string) error { + upper, work := filepath.Join(root, "upper"), filepath.Join(root, "work") + options := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", rt.etcRoot, upper, work) + if err := rt.mountOverlay(rt.etcRoot, options); err != nil { + return fmt.Errorf("mount overlay on %s: %w", rt.etcRoot, err) + } + mountInfo, err := rt.readMountInfo() + if err != nil { + return fmt.Errorf("read mountinfo: %w", err) + } + if !hasOverlayMount(mountInfo, rt.etcRoot, upper, work) { + return errors.New( + "private /etc overlay does not use this fixture's owned upper and work directories", + ) + } + return nil +} + +// installManagedPolicy creates /microsandbox/managed.json holding the empty version-1 policy. +func installManagedPolicy(rt managedPolicyRuntime) (string, error) { + dir := filepath.Join(rt.etcRoot, "microsandbox") + if err := createManagedPolicyDir(rt, dir); err != nil { + return "", err + } + return writeManagedPolicy(rt, dir) +} + +// createManagedPolicyDir requires the directory to be absent from the lower layer, then creates it 0755. +func createManagedPolicyDir(rt managedPolicyRuntime, dir string) error { + _, err := os.Lstat(dir) + if err == nil { + return errors.New( + "managed-policy fixture requires /etc/microsandbox to be absent in the overlay lower layer", + ) + } + if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("inspect %s: %w", dir, err) + } + if err := os.Mkdir(dir, 0o755); err != nil { + return fmt.Errorf("create %s: %w", dir, err) + } + return setOwnerMode(dir, rt.uid, rt.gid, 0o755) +} + +// writeManagedPolicy writes the policy to an exclusive temp file and renames it into place. +func writeManagedPolicy(rt managedPolicyRuntime, dir string) (string, error) { + tmp := filepath.Join(dir, fmt.Sprintf(".managed.json.initial.%d", os.Getpid())) + file, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return "", fmt.Errorf("write initial managed policy: %w", err) + } + _, writeErr := file.WriteString(managedPolicyDocument) + if err := errors.Join(writeErr, file.Close()); err != nil { + return "", fmt.Errorf("write initial managed policy: %w", err) + } + if err := setOwnerMode(tmp, rt.uid, rt.gid, 0o600); err != nil { + return "", err + } + policy := filepath.Join(dir, "managed.json") + if err := os.Rename(tmp, policy); err != nil { + return "", fmt.Errorf("install managed policy: %w", err) + } + return policy, nil +} + +func setOwnerMode(path string, uid, gid uint32, mode os.FileMode) error { + if err := os.Chown(path, int(uid), int(gid)); err != nil { + return fmt.Errorf("chown %s: %w", path, err) + } + if err := os.Chmod(path, mode); err != nil { + return fmt.Errorf("chmod %s: %w", path, err) + } + return nil +} + +// hasOverlayMount reports whether one mountinfo line shows an overlay at mountPoint whose super +// options name exactly the upper and work directories. Both must share a line. +func hasOverlayMount(mountInfo, mountPoint, upper, work string) bool { + for line := range strings.SplitSeq(mountInfo, "\n") { + options := overlaySuperOptions(line, mountPoint) + if options != nil && slices.Contains(options, "upperdir="+upper) && + slices.Contains(options, "workdir="+work) { + return true + } + } + return false +} + +// overlaySuperOptions returns the super options of a mountinfo line that is an overlay at mountPoint, +// or nil. The single "-" separator splits optional fields from the fstype and source fields. +func overlaySuperOptions(line, mountPoint string) []string { + fields := strings.Fields(line) + sep := slices.Index(fields, "-") + if len(fields) < 5 || fields[4] != mountPoint || sep < 0 || sep+3 >= len(fields) || + fields[sep+1] != "overlay" { + return nil + } + return strings.Split(fields[sep+3], ",") +} + +// managedPolicyChildEnvironment replaces every managed-policy key in environ with the values +// the e2e test binary expects. +func managedPolicyChildEnvironment( + environ []string, + ns managedPolicyNamespace, + policy string, +) []string { + overrides := []string{ + managedPolicyMarker + "=true", + "DEVSY_TEST_MSB_MANAGED_POLICY=" + policy, + "DEVSY_TEST_MSB_FIXTURE_ROOT=" + ns.root, + "DEVSY_TEST_MSB_PARENT_MNT_NS=" + ns.parent, + "DEVSY_TEST_MSB_EXPECTED_MNT_NS=" + ns.current, + "DEVSY_TEST_MSB_EXPECTED_UPPER=" + filepath.Join(ns.root, "upper"), + "DEVSY_TEST_MSB_EXPECTED_WORK=" + filepath.Join(ns.root, "work"), + } + kept := slices.DeleteFunc(slices.Clone(environ), func(entry string) bool { + key, _, _ := strings.Cut(entry, "=") + return slices.ContainsFunc( + overrides, + func(o string) bool { return strings.HasPrefix(o, key+"=") }, + ) + }) + return append(kept, overrides...) +} diff --git a/hack/microsandbox/managed_policy_other.go b/hack/microsandbox/managed_policy_other.go new file mode 100644 index 0000000000..d1a43847f6 --- /dev/null +++ b/hack/microsandbox/managed_policy_other.go @@ -0,0 +1,23 @@ +//go:build !linux + +package main + +import ( + "fmt" + "io" + "runtime" +) + +// managedPolicyE2ECLI rejects managed-policy-e2e off Linux, which needs mount namespaces and overlayfs. +func managedPolicyE2ECLI(_ []string, stderr io.Writer) int { + _, _ = fmt.Fprintf( + stderr, + "managed-policy-e2e is unsupported on %s: requires Linux mount namespaces and overlayfs\n", + runtime.GOOS, + ) + return 2 +} + +func managedPolicyNamespaceCLI(args []string, stderr io.Writer) int { + return managedPolicyE2ECLI(args, stderr) +} diff --git a/hack/microsandbox/managed_policy_test.go b/hack/microsandbox/managed_policy_test.go new file mode 100644 index 0000000000..00b1df66b9 --- /dev/null +++ b/hack/microsandbox/managed_policy_test.go @@ -0,0 +1,351 @@ +//go:build linux + +package main + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +const ( + managedPolicyParentNS = "mnt:[parent]" + managedPolicyChildNS = "mnt:[child]" + managedPolicyPathEnv = "PATH=/bin" + managedPolicyTestArg = "-test.v" + managedPolicyEnabled = "true" +) + +// managedPolicyFake records privileged calls instead of performing them. +type managedPolicyFake struct { + namespace string + mountInfo string + options string + exec string + argv []string + env []string +} + +// newManagedPolicyFake returns a root-like runtime whose ownership checks match the test user. +func newManagedPolicyFake(etc string) (managedPolicyRuntime, *managedPolicyFake) { + fake := &managedPolicyFake{namespace: managedPolicyParentNS} + rt := managedPolicyRuntime{ + euid: 0, + uid: uint32(os.Getuid()), + gid: uint32(os.Getgid()), + etcRoot: etc, + mountNamespace: func() (string, error) { return fake.namespace, nil }, + mountOverlay: func(_, options string) error { + fake.options = options + return nil + }, + readMountInfo: func() (string, error) { return fake.mountInfo, nil }, + lookPath: func(name string) (string, error) { return "/usr/bin/" + name, nil }, + executable: func() (string, error) { return "/bin/e2e-wrapper", nil }, + replace: func(path string, argv, env []string) error { + fake.exec, fake.argv, fake.env = path, argv, env + return nil + }, + } + return rt, fake +} + +func overlayMountInfo(mountPoint, upper, work string) string { + return fmt.Sprintf( + "36 25 0:31 / %s rw,relatime - overlay overlay rw,lowerdir=/etc,upperdir=%s,workdir=%s", + mountPoint, + upper, + work, + ) +} + +func requireManagedPolicyMode(t *testing.T, path string, want os.FileMode) { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != want { + t.Fatalf("%s mode = %o, want %o", path, got, want) + } +} + +// requireUnshareInvocation checks the argv that stage 1 hands to unshare. +func requireUnshareInvocation(t *testing.T, argv []string, runnerTemp string, args []string) { + t.Helper() + // argv: 0 unshare, 1-4 flags, 5 self, 6 namespace cmd, 7 root, 8 id file, 9 parent, 10+ args. + if len(argv) < 10 || argv[6] != managedPolicyNamespaceCommand || + argv[9] != managedPolicyParentNS { + t.Fatalf("argv = %q", argv) + } + if !slices.Equal(argv[10:], args) { + t.Fatalf("forwarded args = %q, want %q", argv[10:], args) + } + requireFixtureRoot(t, argv[7], argv[8], runnerTemp) +} + +// requireFixtureRoot checks the fixture lies under RUNNER_TEMP, the id file is inside it, +// and its directories are 0700. +func requireFixtureRoot(t *testing.T, root, idFile, runnerTemp string) { + t.Helper() + if filepath.Dir(root) != runnerTemp || + !strings.HasPrefix(filepath.Base(root), "devsy-managed-policy.") { + t.Fatalf("fixture root %q not under RUNNER_TEMP", root) + } + if idFile != filepath.Join(root, managedPolicyIDFile) { + t.Fatalf("id file = %q", idFile) + } + for _, dir := range []string{root, filepath.Join(root, "upper"), filepath.Join(root, "work")} { + requireManagedPolicyMode(t, dir, 0o700) + } +} + +// requireNamespaceID checks the id file written by stage 2 holds the child namespace. +func requireNamespaceID(t *testing.T, idFile, want string) { + t.Helper() + id, err := os.ReadFile(idFile) + if err != nil || string(id) != want+"\n" { + t.Fatalf("id file = %q, %v", id, err) + } + requireManagedPolicyMode(t, idFile, 0o600) +} + +// requireInstalledPolicy checks the initial policy file and its directory. +func requireInstalledPolicy(t *testing.T, policy string) { + t.Helper() + data, err := os.ReadFile(policy) + if err != nil || string(data) != managedPolicyDocument { + t.Fatalf("policy = %q, %v", data, err) + } + requireManagedPolicyMode(t, policy, 0o600) + requireManagedPolicyMode(t, filepath.Dir(policy), 0o755) +} + +func TestManagedPolicyE2EInputInvariants(t *testing.T) { + valid := t.TempDir() + good := []string{managedPolicyTestBinary} + cases := []struct { + name, marker, runnerTemp, want string + euid int + args []string + }{ + { + "non-root", managedPolicyEnabled, valid, + "managed-policy fixture must run as root", 1000, good, + }, + { + "missing marker", "", valid, + "managed-policy E2E wrapper requires the CI expectation marker", 0, good, + }, + { + "missing runner temp", managedPolicyEnabled, filepath.Join(valid, "absent"), + "RUNNER_TEMP must name an existing directory", 0, good, + }, + { + "wrong binary", managedPolicyEnabled, valid, + "expected the already-built ./e2e.test binary as the first argument", 0, + []string{"./other.test"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(managedPolicyMarker, tc.marker) + t.Setenv("RUNNER_TEMP", tc.runnerTemp) + rt, fake := newManagedPolicyFake(t.TempDir()) + rt.euid = tc.euid + err := runManagedPolicyE2E(rt, tc.args) + if err == nil || err.Error() != tc.want { + t.Fatalf("error = %v, want %q", err, tc.want) + } + if fake.exec != "" { + t.Fatal("re-exec must not happen on invalid input") + } + }) + } +} + +func TestManagedPolicyE2EReexecsInFixtureUnderRunnerTemp(t *testing.T) { + runnerTemp := t.TempDir() + t.Setenv(managedPolicyMarker, managedPolicyEnabled) + t.Setenv("RUNNER_TEMP", runnerTemp) + rt, fake := newManagedPolicyFake(t.TempDir()) + args := []string{managedPolicyTestBinary, managedPolicyTestArg} + if err := runManagedPolicyE2E(rt, args); err != nil { + t.Fatal(err) + } + if fake.exec != "/usr/bin/unshare" { + t.Fatalf("exec = %q, want /usr/bin/unshare", fake.exec) + } + requireUnshareInvocation(t, fake.argv, runnerTemp, args) +} + +func TestCreateManagedPolicyFixtureRejectsUnexpectedOwner(t *testing.T) { + _, err := createManagedPolicyFixture(t.TempDir(), uint32(os.Getuid())+1, uint32(os.Getgid())) + if err == nil || !strings.Contains(err.Error(), "unexpected ownership or mode") { + t.Fatalf("error = %v", err) + } +} + +func TestManagedPolicyNamespaceInstallsPolicyAndExecs(t *testing.T) { + root, etc := t.TempDir(), t.TempDir() + rt, fake := newManagedPolicyFake(etc) + fake.namespace = managedPolicyChildNS + upper, work := filepath.Join(root, "upper"), filepath.Join(root, "work") + fake.mountInfo = overlayMountInfo(etc, upper, work) + idFile := filepath.Join(root, managedPolicyIDFile) + command := []string{managedPolicyTestBinary, managedPolicyTestArg} + args := append([]string{root, idFile, managedPolicyParentNS}, command...) + if err := runManagedPolicyNamespace(rt, args); err != nil { + t.Fatal(err) + } + requireNamespaceID(t, idFile, managedPolicyChildNS) + policy := filepath.Join(etc, "microsandbox", "managed.json") + requireInstalledPolicy(t, policy) + if !strings.Contains(fake.options, "upperdir="+upper) || + !strings.Contains(fake.options, "workdir="+work) { + t.Fatalf("overlay options = %q", fake.options) + } + if fake.exec != managedPolicyTestBinary || !slices.Equal(fake.argv, command) { + t.Fatalf("exec = %q argv = %q", fake.exec, fake.argv) + } + if !slices.Contains(fake.env, "DEVSY_TEST_MSB_MANAGED_POLICY="+policy) { + t.Fatal("child env missing policy path") + } +} + +// namespaceViolation is one stage-2 invariant case; setup prepares the fake runtime and etc tree. +type namespaceViolation struct { + name, want string + args []string + setup func(t *testing.T, etc string, rt *managedPolicyRuntime, fake *managedPolicyFake) +} + +func runNamespaceViolations(t *testing.T, cases []namespaceViolation) { + t.Helper() + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + etc := t.TempDir() + rt, fake := newManagedPolicyFake(etc) + tc.setup(t, etc, &rt, fake) + err := runManagedPolicyNamespace(rt, tc.args) + if err == nil || err.Error() != tc.want { + t.Fatalf("error = %v, want %q", err, tc.want) + } + if fake.exec != "" { + t.Fatal("re-exec must not happen on invariant violation") + } + }) + } +} + +func TestManagedPolicyNamespaceRejectsInvariantViolations(t *testing.T) { + root := t.TempDir() + upper, work := filepath.Join(root, "upper"), filepath.Join(root, "work") + args := []string{ + root, + filepath.Join(root, managedPolicyIDFile), + managedPolicyParentNS, + managedPolicyTestBinary, + } + noSetup := func(*testing.T, string, *managedPolicyRuntime, *managedPolicyFake) {} + runNamespaceViolations(t, []namespaceViolation{ + { + name: "not root", want: "managed-policy namespace must run as root", args: args, + setup: func(_ *testing.T, _ string, rt *managedPolicyRuntime, _ *managedPolicyFake) { + rt.euid = 1000 + }, + }, + { + name: "too few args", + setup: noSetup, + args: args[:3], + want: "managed-policy-namespace requires ", + }, + { + name: "namespace unchanged", + want: "mount namespace did not change", + args: args, + setup: noSetup, + }, + { + name: "foreign overlay", + args: args, + setup: func(_ *testing.T, etc string, _ *managedPolicyRuntime, fake *managedPolicyFake) { + fake.namespace = managedPolicyChildNS + fake.mountInfo = overlayMountInfo(etc, "/elsewhere/upper", work) + }, + want: "private /etc overlay does not use this fixture's owned upper and work directories", + }, + { + name: "existing microsandbox dir", + args: args, + want: "managed-policy fixture requires /etc/microsandbox to be absent in the overlay lower layer", + setup: func(t *testing.T, etc string, _ *managedPolicyRuntime, fake *managedPolicyFake) { + fake.namespace = managedPolicyChildNS + fake.mountInfo = overlayMountInfo(etc, upper, work) + if err := os.Mkdir(filepath.Join(etc, "microsandbox"), 0o755); err != nil { + t.Fatal(err) + } + }, + }, + }) +} + +func TestHasOverlayMount(t *testing.T) { + upper, work := "/fixture/upper", "/fixture/work" + cases := []struct { + name, mountInfo string + want bool + }{ + {"matching overlay", overlayMountInfo("/etc", upper, work), true}, + { + "upper and work on different lines", + "36 25 0:31 / /etc rw - overlay overlay rw,upperdir=" + upper + "\n" + + "37 25 0:32 / /etc rw - overlay overlay rw,workdir=" + work, false, + }, + {"other mount point", overlayMountInfo("/other", upper, work), false}, + { + "non-overlay fstype", + "36 25 8:1 / /etc rw - ext4 /dev/sda1 rw,upperdir=" + upper + ",workdir=" + work, + false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := hasOverlayMount(tc.mountInfo, "/etc", upper, work); got != tc.want { + t.Fatalf("hasOverlayMount = %v, want %v", got, tc.want) + } + }) + } +} + +func TestManagedPolicyChildEnvironmentReplacesManagedKeys(t *testing.T) { + ns := managedPolicyNamespace{ + root: "/fixture", + parent: managedPolicyParentNS, + current: managedPolicyChildNS, + } + environ := []string{ + managedPolicyPathEnv, + "DEVSY_TEST_MSB_EXPECTED_UPPER=/stale/upper", + managedPolicyMarker + "=false", + } + env := managedPolicyChildEnvironment(environ, ns, "/etc/microsandbox/managed.json") + for _, want := range []string{ + managedPolicyPathEnv, + managedPolicyMarker + "=true", + "DEVSY_TEST_MSB_EXPECTED_UPPER=/fixture/upper", + "DEVSY_TEST_MSB_MANAGED_POLICY=/etc/microsandbox/managed.json", + } { + if !slices.Contains(env, want) { + t.Errorf("child env missing %q", want) + } + } + if slices.Contains(env, "DEVSY_TEST_MSB_EXPECTED_UPPER=/stale/upper") || + slices.Contains(env, managedPolicyMarker+"=false") { + t.Error("stale managed-policy value kept") + } +} diff --git a/hack/microsandbox/run.go b/hack/microsandbox/run.go index 9f863447a5..41151364fd 100644 --- a/hack/microsandbox/run.go +++ b/hack/microsandbox/run.go @@ -93,6 +93,11 @@ func executeSuite( _, _ = fmt.Fprintf(output.stderr, "::error::%v\n", err) return suiteExecution{testStatus: 2} } + prefix, err := suiteCommandPrefix(cfg.label) + if err != nil { + _, _ = fmt.Fprintf(output.stderr, "::error::%v\n", err) + return suiteExecution{testStatus: 2} + } reportPath := filepath.Join(cfg.tempDir, "microsandbox-"+cfg.label+".json") logPath := filepath.Join(cfg.tempDir, "microsandbox-"+cfg.label+".log") @@ -111,9 +116,7 @@ func executeSuite( "--ginkgo.label-filter=" + cfg.label, "-ginkgo.json-report=" + reportPath, } - if cfg.label == managedPolicyLabel { - commandArgs = append([]string{"../hack/run-managed-policy-e2e.sh"}, commandArgs...) - } + commandArgs = append(prefix, commandArgs...) args = append(args, commandArgs...) cmd := exec.CommandContext(ctx, "sudo", args...) cmd.Dir = filepath.Join(cfg.repositoryDir, "e2e") @@ -181,6 +184,23 @@ func isValidSuiteLabel(label string) bool { return suiteLabelPattern.MatchString(label) } +// currentExecutable resolves this binary for the managed-policy re-exec. It is a +// variable so tests can make resolution fail. +var currentExecutable = os.Executable + +// suiteCommandPrefix returns the argv prefix that routes the managed-policy suite +// through this binary's managed-policy-e2e subcommand. Other labels get no prefix. +func suiteCommandPrefix(label string) ([]string, error) { + if label != managedPolicyLabel { + return nil, nil + } + exe, err := currentExecutable() + if err != nil { + return nil, fmt.Errorf("resolve managed-policy wrapper: %w", err) + } + return []string{exe, managedPolicyE2ECommand}, nil +} + func suiteSudoArguments(cfg suiteConfig) ([]string, error) { values, err := requiredSuiteEnvironment() if err != nil { diff --git a/hack/microsandbox/run_test.go b/hack/microsandbox/run_test.go index 9c8fa884d1..96c9d1e12d 100644 --- a/hack/microsandbox/run_test.go +++ b/hack/microsandbox/run_test.go @@ -116,7 +116,16 @@ func TestRunSuiteManagedPolicyUsesWrapperAndFixtureEnvironment(t *testing.T) { ); code != 0 { t.Fatalf("runSuite() = %d; stderr=%s", code, stderr.String()) } - if len(commands) != 2 || !stringArgsContain(commands[0], "../hack/run-managed-policy-e2e.sh") { + assertManagedPolicyInvocation(t, commands, tempDir) +} + +func assertManagedPolicyInvocation(t *testing.T, commands [][]string, tempDir string) { + t.Helper() + exe, exeErr := os.Executable() + if exeErr != nil { + t.Fatal(exeErr) + } + if len(commands) != 2 || !isManagedPolicyInvocation(commands[0], exe) { t.Fatalf("managed invocation = %#v", commands) } for _, want := range []string{ @@ -128,6 +137,38 @@ func TestRunSuiteManagedPolicyUsesWrapperAndFixtureEnvironment(t *testing.T) { } } +// isManagedPolicyInvocation reports whether argv runs exe with the managed-policy +// wrapper command followed by ./e2e.test. +func isManagedPolicyInvocation(argv []string, exe string) bool { + index := slices.Index(argv, exe) + return index >= 0 && index+2 < len(argv) && + argv[index+1] == managedPolicyE2ECommand && argv[index+2] == "./e2e.test" +} + +func TestRunSuiteManagedPolicyReportsExecutableResolutionFailureBeforeInvocation(t *testing.T) { + cfg, _ := newSuiteRunFixture(t, managedPolicyLabel) + original := currentExecutable + currentExecutable = func() (string, error) { return "", errors.New("no executable path") } + t.Cleanup(func() { currentExecutable = original }) + var stdout, stderr bytes.Buffer + invoked := false + run := func(*exec.Cmd) error { + invoked = true + return nil + } + if code := runSuite( + context.Background(), + cfg, + run, + commandOutput{stdout: &stdout, stderr: &stderr}, + ); code != 2 { + t.Fatalf("runSuite() = %d, want 2; stderr=%s", code, stderr.String()) + } + if invoked { + t.Fatal("managed-policy command ran after executable resolution failed") + } +} + func TestRunSuitePreservesNativeExitPrecedenceAndArtifacts(t *testing.T) { cfg, tempDir := newSuiteRunFixture(t, "up-provider-microsandbox-images") var console, stderr bytes.Buffer diff --git a/hack/run-managed-policy-e2e.sh b/hack/run-managed-policy-e2e.sh deleted file mode 100755 index 9a5366350e..0000000000 --- a/hack/run-managed-policy-e2e.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 077 - -if [[ ${EUID} -ne 0 ]]; then - echo "managed-policy fixture must run as root" >&2 - exit 1 -fi -if [[ ${DEVSY_REQUIRE_MSB_MANAGED_POLICY_FIXTURE:-} != true ]]; then - echo "managed-policy E2E wrapper requires the CI expectation marker" >&2 - exit 1 -fi -if [[ -z ${RUNNER_TEMP:-} || ! -d ${RUNNER_TEMP} ]]; then - echo "RUNNER_TEMP must name an existing directory" >&2 - exit 1 -fi -if [[ ${1:-} != ./e2e.test ]]; then - echo "expected the already-built ./e2e.test binary as the first argument" >&2 - exit 1 -fi - -fixture_root=$(mktemp -d "${RUNNER_TEMP%/}/devsy-managed-policy.XXXXXX") -chmod 700 "$fixture_root" -mkdir -m 700 "$fixture_root/upper" "$fixture_root/work" -for directory in "$fixture_root" "$fixture_root/upper" "$fixture_root/work"; do - if [[ $(stat -c '%u:%g:%a' "$directory") != "0:0:700" ]]; then - echo "managed-policy fixture directory has unexpected ownership or mode: $directory" >&2 - exit 1 - fi -done -namespace_id_file="$fixture_root/mount-namespace-id" -parent_mount_namespace=$(readlink /proc/self/ns/mnt) -export RUNNER_TEMP fixture_root namespace_id_file parent_mount_namespace - -# Deliberately retain the small private scratch under RUNNER_TEMP for runner -# job teardown. Do not remove overlay upper/work after an abnormal namespace -# exit: an unobserved namespace reference could still use them. -unshare --mount --propagation private -- /bin/bash -s -- "$fixture_root" "$namespace_id_file" "$parent_mount_namespace" "$@" <<'NAMESPACE' -set -euo pipefail -umask 077 -fixture_root=$1 -namespace_id_file=$2 -parent_mount_namespace=$3 -shift 3 - -if [[ ${EUID} -ne 0 ]]; then - echo "managed-policy namespace must run as root" >&2 - exit 1 -fi -current_mount_namespace=$(readlink /proc/self/ns/mnt) -if [[ $current_mount_namespace == "$parent_mount_namespace" ]]; then - echo "mount namespace did not change" >&2 - exit 1 -fi -printf '%s\n' "$current_mount_namespace" > "$namespace_id_file" - -# Overlay /etc before creating any managed-policy path so no host file changes. -mount -t overlay overlay -o "lowerdir=/etc,upperdir=$fixture_root/upper,workdir=$fixture_root/work" /etc -if ! awk -v upper="$fixture_root/upper" -v work="$fixture_root/work" ' - $5 == "/etc" { - separator = 0 - for (i = 1; i <= NF; i++) if ($i == "-") separator = i - if (separator && $(separator + 1) == "overlay") { - count = split($(separator + 3), options, ",") - for (i = 1; i <= count; i++) { - if (options[i] == "upperdir=" upper) found_upper = 1 - if (options[i] == "workdir=" work) found_work = 1 - } - if (found_upper && found_work) found = 1 - } - } - END { exit !found } -' /proc/self/mountinfo; then - echo "private /etc overlay does not use this fixture's owned upper and work directories" >&2 - exit 1 -fi -if [[ -e /etc/microsandbox || -L /etc/microsandbox ]]; then - echo "managed-policy fixture requires /etc/microsandbox to be absent in the overlay lower layer" >&2 - exit 1 -fi -install -d -o 0 -g 0 -m 755 /etc/microsandbox -policy_tmp=/etc/microsandbox/.managed.json.initial.$$ -printf '%s\n' '{"version":1,"overrides":{}}' > "$policy_tmp" -chown 0:0 "$policy_tmp" -chmod 600 "$policy_tmp" -mv -T -- "$policy_tmp" /etc/microsandbox/managed.json - -export DEVSY_REQUIRE_MSB_MANAGED_POLICY_FIXTURE=true -export DEVSY_TEST_MSB_MANAGED_POLICY=/etc/microsandbox/managed.json -export DEVSY_TEST_MSB_FIXTURE_ROOT="$fixture_root" -export DEVSY_TEST_MSB_PARENT_MNT_NS="$parent_mount_namespace" -export DEVSY_TEST_MSB_EXPECTED_MNT_NS="$current_mount_namespace" -export DEVSY_TEST_MSB_EXPECTED_UPPER="$fixture_root/upper" -export DEVSY_TEST_MSB_EXPECTED_WORK="$fixture_root/work" -exec "$@" -NAMESPACE