From a35d5259467c7bad21b3368a60c9fd66901fb697 Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:57:23 -0700 Subject: [PATCH 01/49] fix(command): support process liveness and tree termination on Windows Replace the panic("unsupported") stubs compiled on Windows with real implementations. Liveness reuses the Win32 semantics already established in pkg/ide/opener: OpenProcess with PROCESS_QUERY_LIMITED_INFORMATION plus GetExitCodeProcess against STILL_ACTIVE, treating a nonexistent PID as absent while propagating genuine query errors. Termination delegates to taskkill /T /F so a detached task worker is killed together with any descendants that keep owning transport resources, verifying liveness after the kill and tolerating the already-exited race. Refs #1278 --- pkg/command/process_unsupported.go | 11 -- pkg/command/process_windows.go | 108 +++++++++++++++++ pkg/command/process_windows_test.go | 175 ++++++++++++++++++++++++++++ 3 files changed, 283 insertions(+), 11 deletions(-) delete mode 100644 pkg/command/process_unsupported.go create mode 100644 pkg/command/process_windows.go create mode 100644 pkg/command/process_windows_test.go diff --git a/pkg/command/process_unsupported.go b/pkg/command/process_unsupported.go deleted file mode 100644 index eeb7a35d9f..0000000000 --- a/pkg/command/process_unsupported.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build windows - -package command - -func isRunning(pid string) (bool, error) { - panic("unsupported") -} - -func kill(pid string) error { - panic("unsupported") -} diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go new file mode 100644 index 0000000000..87fdd32336 --- /dev/null +++ b/pkg/command/process_windows.go @@ -0,0 +1,108 @@ +//go:build windows + +package command + +import ( + "errors" + "fmt" + "os/exec" + "strconv" + + "golang.org/x/sys/windows" +) + +// stillActive is the exit code GetExitCodeProcess reports for a live process. +const stillActive uint32 = 259 + +func isRunning(pid string) (bool, error) { + parsed, err := parsePID(pid) + if err != nil { + return false, err + } + + handle, err := windows.OpenProcess( + windows.PROCESS_QUERY_LIMITED_INFORMATION, + false, + uint32(parsed), + ) + if err != nil { + // OpenProcess reports a nonexistent PID as ERROR_INVALID_PARAMETER; + // anything else (e.g. access denied) is a real query failure. + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + return false, nil + } + return false, fmt.Errorf("open process %d: %w", parsed, err) + } + defer func() { _ = windows.CloseHandle(handle) }() + + var exitCode uint32 + if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { + return false, fmt.Errorf("query process %d: %w", parsed, err) + } + return exitCode == stillActive, nil +} + +func kill(pid string) error { + parsed, err := parsePID(pid) + if err != nil { + return err + } + + running, err := isRunning(pid) + if err != nil { + return err + } + if !running { + return nil + } + + // /T takes down the worker's descendants as well: a detached up worker + // can spawn children (e.g. devsy workspace ssh --stdio) that keep owning + // transport resources. /F stands in for SIGKILL; Windows has no SIGTERM + // equivalent for arbitrary processes. + cmd := exec.Command("taskkill", "/PID", strconv.Itoa(parsed), "/T", "/F") + output, runErr := cmd.CombinedOutput() + if runErr != nil { + // The process may have exited on its own between the checks. + stillRunning, checkErr := isRunning(pid) + if checkErr == nil && !stillRunning { + return nil + } + return fmt.Errorf( + "taskkill /PID %d /T /F: %w: %s", + parsed, + runErr, + truncateOutput(string(output)), + ) + } + + stillRunning, err := isRunning(pid) + if err != nil { + return err + } + if stillRunning { + return fmt.Errorf("taskkill /PID %d /T /F reported success but the process is still running", parsed) + } + return nil +} + +func parsePID(pid string) (int, error) { + parsed, err := strconv.Atoi(pid) + if err != nil { + return 0, fmt.Errorf("invalid PID %q: %w", pid, err) + } + if parsed <= 0 { + return 0, fmt.Errorf("invalid PID %d: must be positive", parsed) + } + return parsed, nil +} + +// truncateOutput bounds taskkill's captured output; the text itself is +// localized and never parsed. +func truncateOutput(output string) string { + const maxOutput = 256 + if len(output) > maxOutput { + return output[:maxOutput] + "..." + } + return output +} diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go new file mode 100644 index 0000000000..40494fc4fe --- /dev/null +++ b/pkg/command/process_windows_test.go @@ -0,0 +1,175 @@ +//go:build windows + +package command + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +const ( + helperEnvMarker = "DEVSY_TEST_HELPER_PROCESS" + helperEnvPIDFile = "DEVSY_TEST_HELPER_PIDFILE" +) + +// TestHelperProcess is not a test; every helper process re-executes the test +// binary into it. The sleep loop keeps Go's deadlock detector quiet while the +// helper waits to be killed. +func TestHelperProcess(t *testing.T) { + if os.Getenv(helperEnvMarker) != "1" { + return + } + if pidFile := os.Getenv(helperEnvPIDFile); pidFile != "" { + child := exec.Command(os.Args[0], "-test.run", "TestHelperProcess") + child.Env = helperEnv() + if err := child.Start(); err != nil { + os.Exit(1) + } + if err := os.WriteFile( + pidFile, + []byte(strconv.Itoa(child.Process.Pid)), + 0o600, + ); err != nil { + os.Exit(1) + } + } + for { + time.Sleep(time.Hour) + } +} + +// helperEnv strips helper markers from the inherited environment so only the +// process being launched gets them. Windows process creation needs the rest +// of the environment (e.g. SYSTEMROOT) intact. +func helperEnv(extra ...string) []string { + env := []string{} + for _, kv := range os.Environ() { + key, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(key, "DEVSY_TEST_HELPER_") { + continue + } + env = append(env, kv) + } + return append(env, extra...) +} + +func startHelper(t *testing.T, extraEnv ...string) *exec.Cmd { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run", "TestHelperProcess") + cmd.Env = helperEnv(append([]string{helperEnvMarker + "=1"}, extraEnv...)...) + if err := cmd.Start(); err != nil { + t.Fatalf("start helper: %v", err) + } + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + return cmd +} + +func assertRunning(t *testing.T, pid int) { + t.Helper() + running, err := isRunning(strconv.Itoa(pid)) + if err != nil { + t.Fatalf("isRunning(%d): %v", pid, err) + } + if !running { + t.Fatalf("isRunning(%d) = false, want true", pid) + } +} + +func assertNotRunningEventually(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + running, err := isRunning(strconv.Itoa(pid)) + if err == nil && !running { + return + } + time.Sleep(200 * time.Millisecond) + } + t.Fatalf("process %d still running after 15s", pid) +} + +func TestIsRunningDetectsLiveAndExitedProcess(t *testing.T) { + cmd := startHelper(t) + assertRunning(t, cmd.Process.Pid) + + if err := cmd.Process.Kill(); err != nil { + t.Fatalf("kill helper: %v", err) + } + _ = cmd.Wait() + assertNotRunningEventually(t, cmd.Process.Pid) +} + +func TestKillTerminatesProcessTree(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "child.pid") + parent := startHelper(t, helperEnvPIDFile+"="+pidFile) + + var childPID int + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(pidFile) // #nosec G304 -- test-controlled path + if err == nil { + childPID, err = strconv.Atoi(strings.TrimSpace(string(data))) + if err == nil { + break + } + } + time.Sleep(100 * time.Millisecond) + } + if childPID == 0 { + t.Fatal("helper parent never reported its child PID") + } + + assertRunning(t, parent.Process.Pid) + assertRunning(t, childPID) + + if err := Kill(strconv.Itoa(parent.Process.Pid)); err != nil { + t.Fatalf("Kill(parent): %v", err) + } + + assertNotRunningEventually(t, parent.Process.Pid) + assertNotRunningEventually(t, childPID) +} + +func TestKillOnAlreadyExitedProcessIsNoop(t *testing.T) { + cmd := startHelper(t) + if err := cmd.Process.Kill(); err != nil { + t.Fatalf("kill helper: %v", err) + } + _ = cmd.Wait() + assertNotRunningEventually(t, cmd.Process.Pid) + + if err := Kill(strconv.Itoa(cmd.Process.Pid)); err != nil { + t.Errorf("Kill on exited process = %v, want nil", err) + } +} + +func TestKillIsIdempotent(t *testing.T) { + cmd := startHelper(t) + pid := strconv.Itoa(cmd.Process.Pid) + + if err := Kill(pid); err != nil { + t.Fatalf("first Kill: %v", err) + } + if err := Kill(pid); err != nil { + t.Errorf("second Kill = %v, want nil", err) + } +} + +func TestInvalidPIDReturnsError(t *testing.T) { + for _, pid := range []string{"not-a-pid", "0", "-5"} { + if _, err := isRunning(pid); err == nil { + t.Errorf("isRunning(%q) = nil error, want rejection", pid) + } + if err := kill(pid); err == nil { + t.Errorf("kill(%q) = nil, want error", pid) + } + } +} From bfdc066724874ba73a00312577f8d4843f673e60 Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:07:14 -0700 Subject: [PATCH 02/49] fix(task): make cancellation commit after worker termination Task.Cancel previously persisted the terminal canceled state before it knew whether the worker was actually terminated, so a failed or panicking kill left persisted state claiming the operation was gone while its worker kept running, and a retry saw a terminal task and became a no-op. Cancellation is now snapshot, terminate, finalize: a live worker must be confirmed terminated before the canceled state is committed, a kill failure returns an error and leaves the task retryable, a worker that disappears mid-kill satisfies termination, and an accepted cancellation still wins over a result the worker commits while unwinding. The process kill goes through a Store-local seam so the state machine is testable without crashing real processes. Refs #1278 --- pkg/task/export_test.go | 6 ++ pkg/task/store.go | 5 +- pkg/task/task.go | 43 ++++++--- pkg/task/task_pid_windows_test.go | 90 +++++++++++++++++++ pkg/task/task_test.go | 141 ++++++++++++++++++++++++++++++ 5 files changed, 272 insertions(+), 13 deletions(-) create mode 100644 pkg/task/task_pid_windows_test.go diff --git a/pkg/task/export_test.go b/pkg/task/export_test.go index 9ffbc395a5..0e7539de54 100644 --- a/pkg/task/export_test.go +++ b/pkg/task/export_test.go @@ -5,3 +5,9 @@ package task func (s *Store) SetAfterClaimForTest(fn func()) { s.afterClaimForTest = fn } + +// SetKillProcessForTest replaces this store's process termination hook, so a +// test can drive cancellation outcomes deterministically without real PIDs. +func (s *Store) SetKillProcessForTest(fn func(string) error) { + s.killProcess = fn +} diff --git a/pkg/task/store.go b/pkg/task/store.go index 87b3b5185e..ea825557f0 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -9,6 +9,7 @@ import ( "sort" "time" + "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/random" "github.com/gofrs/flock" @@ -20,6 +21,8 @@ const lockTimeout = 5 * time.Second // Store persists task state as one JSON file per task under dir. type Store struct { dir string + // Test seam; see Store.SetKillProcessForTest. + killProcess func(string) error // Test seam; see Store.SetAfterClaimForTest. afterClaimForTest func() } @@ -36,7 +39,7 @@ func NewStoreAt(dir string) (*Store, error) { if err := os.MkdirAll(dir, 0o750); err != nil { return nil, fmt.Errorf("create task dir: %w", err) } - return &Store{dir: dir}, nil + return &Store{dir: dir, killProcess: command.Kill}, nil } func (s *Store) Create(opts CreateOptions) (*Task, error) { diff --git a/pkg/task/task.go b/pkg/task/task.go index 7cf6c82463..026ede3721 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -11,7 +11,6 @@ import ( "time" "github.com/devsy-org/devsy/pkg/clierr" - "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/secrets" "github.com/devsy-org/devsy/pkg/status" @@ -160,30 +159,50 @@ func (t *Task) Succeed(result *config.Result) error { }) } -// Cancel is safe to call even if the process already exited on its own. The -// terminal check and state transition happen atomically under the same -// lock, so a concurrent report from the task's own worker can't race with -// marking it canceled here. +// Cancel terminates the task's worker before recording the task as canceled: +// the persisted terminal state must never claim the operation is gone while +// the worker that performs it is still alive. A kill failure leaves the task +// nonterminal so a later Cancel retries. Cancellation is idempotent: a task +// already terminal, or whose worker already exited, only (re)finalizes the +// canceled state. func (t *Task) Cancel() error { var pid int + observedNonterminal := false err := t.store.update(t.id, func(s *State) { if s.Status.Terminal() { return } + observedNonterminal = true pid = s.PID - s.Status = StatusFailed - s.Error = ErrCanceled.Error() - s.ErrorCode = string(clierr.CodeCanceled) - s.ErrorHint = "Retry the operation when ready." - s.ErrorContext = nil }) if err != nil { return err } - if pid == 0 || !t.store.workerAlive(t.id) { + if !observedNonterminal { return nil } - return command.Kill(strconv.Itoa(pid)) + + if pid != 0 && t.store.workerAlive(t.id) { + if err := t.store.killProcess(strconv.Itoa(pid)); err != nil { + if !t.store.workerAlive(t.id) { + // The worker exited during the kill attempt, so termination + // is satisfied even though the kill itself reported an error. + return t.store.update(t.id, markCanceled) + } + return fmt.Errorf("cancel task %s: terminate worker pid %d: %w", t.id, pid, err) + } + } + // This invocation observed the task as nonterminal, so a result the worker + // committed while unwinding does not win over the accepted cancellation. + return t.store.update(t.id, markCanceled) +} + +func markCanceled(s *State) { + s.Status = StatusFailed + s.Error = ErrCanceled.Error() + s.ErrorCode = string(clierr.CodeCanceled) + s.ErrorHint = "Retry the operation when ready." + s.ErrorContext = nil } // Fail preserves an existing terminal state, so the error a canceled worker diff --git a/pkg/task/task_pid_windows_test.go b/pkg/task/task_pid_windows_test.go new file mode 100644 index 0000000000..318cf2d7e4 --- /dev/null +++ b/pkg/task/task_pid_windows_test.go @@ -0,0 +1,90 @@ +//go:build windows + +package task + +import ( + "os/exec" + "testing" + "time" +) + +// sleepWorker returns a long-running process using only tools a Windows +// runner always has. +func sleepWorker() *exec.Cmd { + return exec.Command("ping", "-n", "30", "127.0.0.1") +} + +func TestCancelSignalsALiveWorkersPID(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + holdWorkerLock(t, tk) + + worker := sleepWorker() + if err := worker.Start(); err != nil { + t.Fatalf("start: %v", err) + } + defer func() { _ = worker.Process.Kill() }() + if err := tk.SetPID(worker.Process.Pid); err != nil { + t.Fatalf("SetPID: %v", err) + } + + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + + waitErr := worker.Wait() + if waitErr == nil { + t.Fatal("worker process exited cleanly, want it terminated by Cancel") + } +} + +func TestCancelDoesNotSignalAProcessThatReusedThePID(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + // Claim then release the worker lock: the kernel frees a dead worker's + // lock the same way, leaving its old PID free for the OS to hand to an + // unrelated process. + if err := tk.HoldWorkerLock(); err != nil { + t.Fatalf("HoldWorkerLock: %v", err) + } + if err := tk.ReleaseWorkerLockForTest(); err != nil { + t.Fatalf("ReleaseWorkerLockForTest: %v", err) + } + + innocent := sleepWorker() + if err := innocent.Start(); err != nil { + t.Fatalf("start: %v", err) + } + defer func() { _ = innocent.Process.Kill() }() + if err := tk.SetPID(innocent.Process.Pid); err != nil { + t.Fatalf("SetPID: %v", err) + } + exited := waitAsync(innocent) + + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + + assertStillRunning(t, exited) +} + +func waitAsync(cmd *exec.Cmd) <-chan error { + exited := make(chan error, 1) + go func() { exited <- cmd.Wait() }() + return exited +} + +func assertStillRunning(t *testing.T, exited <-chan error) { + t.Helper() + select { + case err := <-exited: + t.Fatalf("process exited unexpectedly: %v", err) + case <-time.After(500 * time.Millisecond): + } +} diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index c7bfc0aa63..cfa9288008 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -817,3 +817,144 @@ func TestReconcilePreservesCancellationReason(t *testing.T) { ) } } + +func TestCancelKillFailureRemainsRetryable(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + holdWorkerLock(t, tk) + if err := tk.SetPID(4242); err != nil { + t.Fatalf("SetPID: %v", err) + } + + killCalls := 0 + store.SetKillProcessForTest(func(string) error { + killCalls++ + if killCalls == 1 { + return errors.New("boom") + } + return nil + }) + + if err := tk.Cancel(); err == nil { + t.Fatal("first Cancel = nil, want kill failure") + } + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Status.Terminal() { + t.Fatalf("task recorded terminal %q while its worker survived the failed kill", state.Status) + } + + if err := tk.Cancel(); err != nil { + t.Fatalf("retry Cancel: %v", err) + } + if killCalls != 2 { + t.Errorf("kill invoked %d times, want 2 (one per Cancel)", killCalls) + } + state, err = store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Status != StatusFailed || state.Error != ErrCanceled.Error() || state.ErrorCode != "canceled" { + t.Errorf("unexpected final state: %+v", state) + } +} + +func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + holdWorkerLock(t, tk) + if err := tk.SetPID(4242); err != nil { + t.Fatalf("SetPID: %v", err) + } + + store.SetKillProcessForTest(func(string) error { + state, err := store.Get(tk.ID()) + if err != nil { + t.Errorf("Get during kill: %v", err) + } + if state.Status.Terminal() { + t.Errorf("task recorded %q before its worker was terminated", state.Status) + } + return nil + }) + + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Status != StatusFailed || state.Error != ErrCanceled.Error() { + t.Errorf("unexpected final state: %+v", state) + } +} + +func TestCancelDeadWorkerDoesNotInvokeKill(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := tk.HoldWorkerLock(); err != nil { + t.Fatalf("HoldWorkerLock: %v", err) + } + if err := tk.ReleaseWorkerLockForTest(); err != nil { + t.Fatalf("ReleaseWorkerLockForTest: %v", err) + } + if err := tk.SetPID(4242); err != nil { + t.Fatalf("SetPID: %v", err) + } + + store.SetKillProcessForTest(func(string) error { + t.Error("kill invoked for a task whose worker lock is dead") + return nil + }) + + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Status != StatusFailed || state.Error != ErrCanceled.Error() { + t.Errorf("unexpected final state: %+v", state) + } +} + +func TestCancelRacingLateSuccessStillWinsAfterSuccessfulTermination(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + holdWorkerLock(t, tk) + if err := tk.SetPID(4242); err != nil { + t.Fatalf("SetPID: %v", err) + } + + store.SetKillProcessForTest(func(string) error { + // The worker reports its own success while the kill unwinds it. + return tk.Succeed(&config.Result{}) + }) + + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Status != StatusFailed || state.Error != ErrCanceled.Error() { + t.Errorf("late worker success overwrote the accepted cancellation: %+v", state) + } +} From 65f592f281ba898bc4a283a4f3c1deed5c8cabe5 Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:13:24 -0700 Subject: [PATCH 03/49] fix(workspace): quiesce persisted up tasks before stop and delete Workspace stop/delete correctness depended on Desktop's in-memory activeUpTasks map, which does not survive a Desktop restart, while the CLI stop path never consulted the persisted task store at all. A surviving detached up worker can hold or reacquire lifecycle authority and restart a workspace after it was stopped. The task store now answers ActiveForWorkspace with worker reconciliation, and stop/delete quiesce every active persisted up task through it: before waiting for the workspace lock, and again under it, since task submission and lock acquisition are not atomic. Every matching task is attempted and failures are aggregated; a live worker that cannot be canceled fails the stop/delete instead of being logged and ignored, because proceeding could let the worker recreate deleted resources. Refs #1278 --- cmd/workspace/stop.go | 33 +++++++++ cmd/workspace/stop_test.go | 124 ++++++++++++++++++++++++++++++++++ pkg/task/export_test.go | 6 -- pkg/task/store.go | 38 +++++++++++ pkg/task/store_test.go | 104 ++++++++++++++++++++++++++++ pkg/workspace/delete.go | 35 ++++++++++ pkg/workspace/delete_test.go | 58 ++++++++++++++++ pkg/workspace/quiesce.go | 50 ++++++++++++++ pkg/workspace/quiesce_test.go | 85 +++++++++++++++++++++++ 9 files changed, 527 insertions(+), 6 deletions(-) create mode 100644 cmd/workspace/stop_test.go create mode 100644 pkg/task/store_test.go create mode 100644 pkg/workspace/quiesce.go create mode 100644 pkg/workspace/quiesce_test.go diff --git a/cmd/workspace/stop.go b/cmd/workspace/stop.go index 9a1a12654a..1d70afddaa 100644 --- a/cmd/workspace/stop.go +++ b/cmd/workspace/stop.go @@ -22,6 +22,10 @@ import ( type StopCmd struct { *flags.GlobalFlags client2.StopOptions + // quiesceUpTasks cancels the workspace's persisted detached up tasks; + // nil resolves the default task store. Tests inject it to drive + // cancellation outcomes without real worker processes. + quiesceUpTasks func(workspaceID string) error } // NewStopCmd creates a new destroy command. @@ -123,6 +127,12 @@ func (cmd *StopCmd) run( devsyConfig *config.Config, client client2.BaseWorkspaceClient, ) error { + // Quiesce before waiting for the workspace lock: a still-provisioning up + // worker can be the one holding it. + if err := cmd.quiesce(client.Workspace()); err != nil { + return err + } + // lock workspace if !cmd.Platform.Enabled { err := status.RunStep( @@ -137,6 +147,14 @@ func (cmd *StopCmd) run( defer client.Unlock() } + // Quiesce again under the lock: task submission and lock acquisition are + // not atomic, so a task may have become visible while the lock wait + // blocked. A surviving up worker can restart the workspace after the + // stop, so failing here must fail the stop. + if err := cmd.quiesce(client.Workspace()); err != nil { + return err + } + // get instance status var instanceStatus client2.Status err := status.RunStep( @@ -255,3 +273,18 @@ func otherWorkspaceUsesMachine( } return false } + +func (cmd *StopCmd) quiesce(workspaceID string) error { + quiesce := cmd.quiesceUpTasks + if quiesce == nil { + quiesce = workspace2.QuiesceUpTasksForWorkspace + } + if err := quiesce(workspaceID); err != nil { + return fmt.Errorf( + "cancel detached up tasks for workspace %s: %w", + workspaceID, + err, + ) + } + return nil +} diff --git a/cmd/workspace/stop_test.go b/cmd/workspace/stop_test.go new file mode 100644 index 0000000000..903c7af123 --- /dev/null +++ b/cmd/workspace/stop_test.go @@ -0,0 +1,124 @@ +package workspace + +import ( + "context" + "errors" + "testing" + + "github.com/devsy-org/devsy/cmd/flags" + client2 "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type stopRecordingClient struct { + *fakeWorkspaceClient + calls *[]string +} + +func (c *stopRecordingClient) Stop(ctx context.Context, opts client2.StopOptions) error { + *c.calls = append(*c.calls, "stop") + return c.fakeWorkspaceClient.Stop(ctx, opts) +} + +func newStopTestConfig() *config.Config { + return &config.Config{ + DefaultContext: testContext, + Contexts: map[string]*config.ContextConfig{testContext: {}}, + } +} + +func TestStopCancelsPersistedUpTaskBeforeProviderStop(t *testing.T) { + var calls []string + fake := &fakeWorkspaceClient{ + workspace: testWorkspaceName, + context: testContext, + provider: testProvider, + config: &provider.Workspace{ID: testWorkspaceName, Context: testContext}, + status: client2.StatusRunning, + } + client := &stopRecordingClient{fakeWorkspaceClient: fake, calls: &calls} + cmd := &StopCmd{GlobalFlags: &flags.GlobalFlags{ResultFormat: formatPlain}} + cmd.quiesceUpTasks = func(workspaceID string) error { + assert.Equal(t, testWorkspaceName, workspaceID) + calls = append(calls, "quiesce") + return nil + } + + require.NoError(t, cmd.run(t.Context(), newStopTestConfig(), client)) + + require.NotEmpty(t, calls) + assert.Equal(t, "quiesce", calls[0]) + assert.Contains(t, calls, "stop") + stopIndex := -1 + for i, call := range calls { + if call == "stop" { + stopIndex = i + break + } + } + quiesceBeforeStop := 0 + for _, call := range calls[:stopIndex] { + if call == "quiesce" { + quiesceBeforeStop++ + } + } + assert.Equal(t, 2, quiesceBeforeStop, "stop must quiesce before and after the workspace lock") +} + +func TestStopDoesNotProceedWhenActiveTaskCannotBeKilled(t *testing.T) { + fake := &fakeWorkspaceClient{ + workspace: testWorkspaceName, + context: testContext, + provider: testProvider, + config: &provider.Workspace{ID: testWorkspaceName, Context: testContext}, + status: client2.StatusRunning, + } + var calls []string + client := &stopRecordingClient{fakeWorkspaceClient: fake, calls: &calls} + cmd := &StopCmd{GlobalFlags: &flags.GlobalFlags{ResultFormat: formatPlain}} + cmd.quiesceUpTasks = func(string) error { + return errors.New("boom") + } + + err := cmd.run(t.Context(), newStopTestConfig(), client) + require.Error(t, err) + assert.NotContains(t, calls, "stop") +} + +func TestStopFindsTaskWithoutDesktopState(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + config.ResetPathManager() + t.Cleanup(config.ResetPathManager) + + // A persisted detached up task with no live worker, as left behind after + // a Desktop restart: nothing in memory references it. + store, err := task.NewStore() + require.NoError(t, err) + tk, err := store.Create(task.CreateOptions{Command: "up", WorkspaceID: testWorkspaceName}) + require.NoError(t, err) + + fake := &fakeWorkspaceClient{ + workspace: testWorkspaceName, + context: testContext, + provider: testProvider, + config: &provider.Workspace{ID: testWorkspaceName, Context: testContext}, + status: client2.StatusRunning, + } + var calls []string + client := &stopRecordingClient{fakeWorkspaceClient: fake, calls: &calls} + cmd := &StopCmd{GlobalFlags: &flags.GlobalFlags{ResultFormat: formatPlain}} + + require.NoError(t, cmd.run(t.Context(), newStopTestConfig(), client)) + assert.Contains(t, calls, "stop") + + state, err := store.Get(tk.ID()) + require.NoError(t, err) + assert.Equal(t, task.StatusFailed, state.Status) + assert.Equal(t, task.ErrCanceled.Error(), state.Error) +} diff --git a/pkg/task/export_test.go b/pkg/task/export_test.go index 0e7539de54..9ffbc395a5 100644 --- a/pkg/task/export_test.go +++ b/pkg/task/export_test.go @@ -5,9 +5,3 @@ package task func (s *Store) SetAfterClaimForTest(fn func()) { s.afterClaimForTest = fn } - -// SetKillProcessForTest replaces this store's process termination hook, so a -// test can drive cancellation outcomes deterministically without real PIDs. -func (s *Store) SetKillProcessForTest(fn func(string) error) { - s.killProcess = fn -} diff --git a/pkg/task/store.go b/pkg/task/store.go index ea825557f0..7f23ae3648 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -323,3 +323,41 @@ func syncDir(dir string) { defer func() { _ = d.Close() }() _ = d.Sync() } + +// SetKillProcessForTest replaces this store's process termination hook, so a +// test can drive cancellation outcomes deterministically without real PIDs. +// Production code must never call it. +// +// Not in export_test.go: other packages' tests need it, and a _test.go file is +// only compiled into its own package's test binary. +func (s *Store) SetKillProcessForTest(fn func(string) error) { + s.killProcess = fn +} + +// ActiveForWorkspace returns the non-terminal tasks labeled with the exact +// workspace ID, newest first, reconciling stale workers first so an +// abandoned task does not read as live. A non-empty command restricts the +// match to that command. The store only answers which persisted operations +// are still active; callers decide what to do with them. +func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error) { + states, err := s.List() + if err != nil { + return nil, err + } + + active := make([]*State, 0, len(states)) + for _, state := range states { + if state.WorkspaceID != workspaceID { + continue + } + if command != "" && state.Command != command { + continue + } + state = s.Reconcile(state) + if state.Status.Terminal() { + continue + } + active = append(active, state) + } + return active, nil +} diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go new file mode 100644 index 0000000000..358e9388b1 --- /dev/null +++ b/pkg/task/store_test.go @@ -0,0 +1,104 @@ +package task + +import ( + "testing" + "time" +) + +func createTaskWith(t *testing.T, store *Store, opts CreateOptions) *Task { + t.Helper() + tk, err := store.Create(opts) + if err != nil { + t.Fatalf("Create: %v", err) + } + return tk +} + +func TestActiveForWorkspaceFiltersAndOrders(t *testing.T) { + store := newTestStore(t) + + older := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) + if err := store.update(older.ID(), func(s *State) { + s.StartedAt = s.StartedAt.Add(-time.Hour) + }); err != nil { + t.Fatalf("backdate: %v", err) + } + newer := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) + otherCommand := createTaskWith(t, store, CreateOptions{Command: "build", WorkspaceID: "ws-1"}) + otherWorkspace := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-2"}) + terminal := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) + if err := terminal.Succeed(nil); err != nil { + t.Fatalf("Succeed: %v", err) + } + + active, err := store.ActiveForWorkspace("ws-1", "up") + if err != nil { + t.Fatalf("ActiveForWorkspace: %v", err) + } + if len(active) != 2 { + t.Fatalf("got %d active tasks, want 2: %+v", len(active), active) + } + if active[0].ID != newer.ID() || active[1].ID != older.ID() { + t.Errorf("unexpected order/contents: %s, %s", active[0].ID, active[1].ID) + } + + all, err := store.ActiveForWorkspace("ws-1", "") + if err != nil { + t.Fatalf("ActiveForWorkspace: %v", err) + } + if len(all) != 3 { + t.Errorf("unfiltered match = %d, want 3 (includes %s)", len(all), otherCommand.ID()) + } + + none, err := store.ActiveForWorkspace(otherWorkspace.ID(), "up") + if err != nil { + t.Fatalf("ActiveForWorkspace: %v", err) + } + if len(none) != 0 { + t.Errorf("task id used as workspace id matched %d tasks, want 0", len(none)) + } +} + +func TestActiveForWorkspaceReconcilesAbandonedWorkers(t *testing.T) { + store := newTestStore(t) + tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) + + // A worker that claimed then lost its lock died without recording a + // result; the query must reconcile it instead of reporting it active. + if err := tk.HoldWorkerLock(); err != nil { + t.Fatalf("HoldWorkerLock: %v", err) + } + if err := tk.ReleaseWorkerLockForTest(); err != nil { + t.Fatalf("ReleaseWorkerLockForTest: %v", err) + } + + active, err := store.ActiveForWorkspace("ws-1", "up") + if err != nil { + t.Fatalf("ActiveForWorkspace: %v", err) + } + if len(active) != 0 { + t.Errorf("abandoned task reported active: %+v", active) + } + + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if !state.Status.Terminal() { + t.Errorf("reconciled task status = %q, want terminal", state.Status) + } +} + +func TestActiveForWorkspaceTreatsLiveWorkerAsActive(t *testing.T) { + store := newTestStore(t) + tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) + holdWorkerLock(t, tk) + + active, err := store.ActiveForWorkspace("ws-1", "up") + if err != nil { + t.Fatalf("ActiveForWorkspace: %v", err) + } + if len(active) != 1 || active[0].ID != tk.ID() { + t.Errorf("live worker task not reported active: %+v", active) + } +} diff --git a/pkg/workspace/delete.go b/pkg/workspace/delete.go index 3745424387..219e73da64 100644 --- a/pkg/workspace/delete.go +++ b/pkg/workspace/delete.go @@ -26,6 +26,10 @@ type DeleteOptions struct { Force bool ClientDelete client2.DeleteOptions Owner platform.OwnerFilter + // quiesceUpTasks cancels the workspace's persisted detached up tasks; + // nil resolves the default task store. Tests inject it to drive + // cancellation outcomes without real worker processes. + quiesceUpTasks func(workspaceID string) error } // Delete deletes a workspace, handling imported workspaces, single-machine @@ -56,6 +60,12 @@ func Delete(ctx context.Context, opts DeleteOptions) (string, error) { defer opener.KillBrowserTunnel(client.Context(), client.Workspace()) + // A surviving detached up worker can recreate resources during or after + // the delete, so quiescence failure blocks even a forced delete. + if err := opts.quiesce(client.Workspace()); err != nil { + return "", err + } + if !opts.Force && client.WorkspaceConfig().Imported { var id string err := progress.RunStep( @@ -77,6 +87,12 @@ func Delete(ctx context.Context, opts DeleteOptions) (string, error) { } defer unlock() + // Re-scan under the lock: a task may have become visible while lock + // acquisition waited. + if err := opts.quiesce(client.Workspace()); err != nil { + return "", err + } + stopIfRunning(ctx, client, status) var id string @@ -243,6 +259,10 @@ func handleDeleteLoadError( func forceDeleteFolder(opts DeleteOptions, workspaceID string) (string, error) { log.Errorf("error retrieving workspace, force-deleting folder") + if err := opts.quiesce(workspaceID); err != nil { + return "", err + } + err := clientimplementation.DeleteWorkspaceFolder( clientimplementation.DeleteWorkspaceFolderParams{ Context: opts.DevsyConfig.DefaultContext, @@ -465,3 +485,18 @@ func removeIfContentOrphan(contextName, contentsDir string, entry os.DirEntry) { } log.Debugf("removed orphan content dir with no matching workspace: workspace=%s", entry.Name()) } + +func (opts DeleteOptions) quiesce(workspaceID string) error { + quiesce := opts.quiesceUpTasks + if quiesce == nil { + quiesce = QuiesceUpTasksForWorkspace + } + if err := quiesce(workspaceID); err != nil { + return fmt.Errorf( + "cancel detached up tasks for workspace %s: %w", + workspaceID, + err, + ) + } + return nil +} diff --git a/pkg/workspace/delete_test.go b/pkg/workspace/delete_test.go index dd07b9e5e0..36409f165e 100644 --- a/pkg/workspace/delete_test.go +++ b/pkg/workspace/delete_test.go @@ -2,6 +2,7 @@ package workspace import ( "context" + "errors" "os" "path/filepath" "testing" @@ -9,6 +10,7 @@ import ( "github.com/devsy-org/devsy/pkg/client" "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/task" "github.com/stretchr/testify/require" ) @@ -113,3 +115,59 @@ func (c *preResolvedDeleteClient) Delete(context.Context, client.DeleteOptions) c.deleted = true return nil } + +func TestDeleteQuiescesPersistedUpTaskBeforeDeleting(t *testing.T) { + setupTestPathManager(t) + + store, err := task.NewStore() + require.NoError(t, err) + tk, err := store.Create(task.CreateOptions{Command: "up", WorkspaceID: "chosen"}) + require.NoError(t, err) + + fake := &preResolvedDeleteClient{ + workspaceID: "chosen", + config: &provider.Workspace{ID: "chosen", Context: testDefaultContext}, + } + devsyConfig := &config.Config{ + DefaultContext: testDefaultContext, + Contexts: map[string]*config.ContextConfig{testDefaultContext: {}}, + } + + id, err := Delete(t.Context(), DeleteOptions{ + DevsyConfig: devsyConfig, + Client: fake, + Force: true, + }) + require.NoError(t, err) + require.Equal(t, "chosen", id) + require.True(t, fake.deleted) + + state, err := store.Get(tk.ID()) + require.NoError(t, err) + require.Equal(t, task.StatusFailed, state.Status) + require.Equal(t, task.ErrCanceled.Error(), state.Error) +} + +func TestDeleteBlockedWhenActiveUpTaskCannotBeQuiesced(t *testing.T) { + setupTestPathManager(t) + + fake := &preResolvedDeleteClient{ + workspaceID: "chosen", + config: &provider.Workspace{ID: "chosen", Context: testDefaultContext}, + } + devsyConfig := &config.Config{ + DefaultContext: testDefaultContext, + Contexts: map[string]*config.ContextConfig{testDefaultContext: {}}, + } + + _, err := Delete(t.Context(), DeleteOptions{ + DevsyConfig: devsyConfig, + Client: fake, + Force: true, + quiesceUpTasks: func(string) error { + return errors.New("boom") + }, + }) + require.Error(t, err) + require.False(t, fake.deleted, "delete must not proceed while a live up task survives") +} diff --git a/pkg/workspace/quiesce.go b/pkg/workspace/quiesce.go new file mode 100644 index 0000000000..f662f7036a --- /dev/null +++ b/pkg/workspace/quiesce.go @@ -0,0 +1,50 @@ +package workspace + +import ( + "errors" + "fmt" + + "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/task" +) + +// upTaskCommand is the command label detached up workers are created with. +const upTaskCommand = "up" + +// QuiesceUpTasks cancels every persisted, still-active detached up task for +// the workspace so no surviving worker can mutate or restart it during +// stop/delete. Every matching task is attempted even when one fails; +// failures are aggregated, never logged and ignored, because a surviving +// worker is a known local mutator capable of reversing the lifecycle change. +func QuiesceUpTasks(store *task.Store, workspaceID string) error { + active, err := store.ActiveForWorkspace(workspaceID, upTaskCommand) + if err != nil { + return fmt.Errorf("list active up tasks for workspace %s: %w", workspaceID, err) + } + + var errs []error + for _, state := range active { + if err := store.Open(state.ID).Cancel(); err != nil { + errs = append(errs, fmt.Errorf("cancel up task %s: %w", state.ID, err)) + } + } + if len(active) > 0 { + log.Debugf( + "quiesced up tasks for workspace %s: tasks=%d failures=%d", + workspaceID, + len(active), + len(errs), + ) + } + return errors.Join(errs...) +} + +// QuiesceUpTasksForWorkspace resolves the default task store and quiesces +// the workspace's active up tasks. +func QuiesceUpTasksForWorkspace(workspaceID string) error { + store, err := task.NewStore() + if err != nil { + return fmt.Errorf("open task store: %w", err) + } + return QuiesceUpTasks(store, workspaceID) +} diff --git a/pkg/workspace/quiesce_test.go b/pkg/workspace/quiesce_test.go new file mode 100644 index 0000000000..475a77a8fe --- /dev/null +++ b/pkg/workspace/quiesce_test.go @@ -0,0 +1,85 @@ +package workspace + +import ( + "errors" + "testing" + + "github.com/devsy-org/devsy/pkg/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTaskStore(t *testing.T) *task.Store { + t.Helper() + store, err := task.NewStoreAt(t.TempDir()) + require.NoError(t, err) + return store +} + +func createUpTask(t *testing.T, store *task.Store, workspaceID string) *task.Task { + t.Helper() + tk, err := store.Create(task.CreateOptions{Command: "up", WorkspaceID: workspaceID}) + require.NoError(t, err) + return tk +} + +func taskState(t *testing.T, store *task.Store, id string) *task.State { + t.Helper() + state, err := store.Get(id) + require.NoError(t, err) + return state +} + +func TestQuiesceUpTasksCancelsOnlyActiveUpTasksForWorkspace(t *testing.T) { + store := newTaskStore(t) + targetA := createUpTask(t, store, "ws-1") + targetB := createUpTask(t, store, "ws-1") + terminal := createUpTask(t, store, "ws-1") + require.NoError(t, terminal.Succeed(nil)) + otherWorkspace := createUpTask(t, store, "ws-2") + otherCommand, err := store.Create(task.CreateOptions{Command: "build", WorkspaceID: "ws-1"}) + require.NoError(t, err) + + require.NoError(t, QuiesceUpTasks(store, "ws-1")) + + for _, id := range []string{targetA.ID(), targetB.ID()} { + state := taskState(t, store, id) + assert.Equal(t, task.StatusFailed, state.Status, "task %s", id) + assert.Equal(t, task.ErrCanceled.Error(), state.Error, "task %s", id) + } + assert.Equal(t, task.StatusSucceeded, taskState(t, store, terminal.ID()).Status) + assert.Equal(t, task.StatusPending, taskState(t, store, otherWorkspace.ID()).Status) + assert.Equal(t, task.StatusPending, taskState(t, store, otherCommand.ID()).Status) +} + +func TestQuiesceUpTasksAttemptsEveryTaskAndAggregatesFailures(t *testing.T) { + store := newTaskStore(t) + failing := createUpTask(t, store, "ws-1") + succeeding := createUpTask(t, store, "ws-1") + + // Both workers read as live; one kill keeps failing. + for _, tk := range []*task.Task{failing, succeeding} { + require.NoError(t, tk.HoldWorkerLock()) + tk := tk + t.Cleanup(func() { _ = tk.ReleaseWorkerLockForTest() }) + } + require.NoError(t, failing.SetPID(1111)) + require.NoError(t, succeeding.SetPID(2222)) + store.SetKillProcessForTest(func(pid string) error { + if pid == "1111" { + return errors.New("boom") + } + return nil + }) + + err := QuiesceUpTasks(store, "ws-1") + require.Error(t, err) + assert.Contains(t, err.Error(), failing.ID()) + + // The failing task stays retryable while the other task was still + // attempted and canceled. + assert.False(t, taskState(t, store, failing.ID()).Status.Terminal()) + state := taskState(t, store, succeeding.ID()) + assert.Equal(t, task.StatusFailed, state.Status) + assert.Equal(t, task.ErrCanceled.Error(), state.Error) +} From e44c86a4bb0e61e5d23aca2b3361ceb524dd2900 Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:18:53 -0700 Subject: [PATCH 04/49] fix(tunnel): make local tunnel health checks side-effect-free The local tunnel's periodic health probe dialed through the same workspace ssh --stdio path as real traffic, and that path is allowed to auto-start a stopped workspace, so a background health check could reverse a deliberate stop and keep reviving the container. LocalTunnelOptions now separates the planes: DialFunc serves only real accepted connections, and an optional HealthCheckFunc provides strictly observational liveness probing; with no health function the listener lifetime simply follows its context. The workspace tunnel supplies a status-based probe that reports healthy only when the workspace is Running and never touches the SSH/start path, so an externally stopped workspace now tears the tunnel down instead of being restarted. Refs #1278 --- cmd/workspace/up/tunnel.go | 24 +++++++- cmd/workspace/up/tunnel_test.go | 53 ++++++++++++++++++ pkg/tunnel/local_listener.go | 20 +++++-- pkg/tunnel/local_listener_test.go | 93 +++++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 cmd/workspace/up/tunnel_test.go diff --git a/cmd/workspace/up/tunnel.go b/cmd/workspace/up/tunnel.go index 69b7cb5c23..82ce300261 100644 --- a/cmd/workspace/up/tunnel.go +++ b/cmd/workspace/up/tunnel.go @@ -10,6 +10,25 @@ import ( "github.com/devsy-org/devsy/pkg/tunnel" ) +// workspaceTunnelHealth probes whether the workspace behind the tunnel is +// still running. It is strictly observational: it must never dial the SSH +// path or otherwise start the workspace, because a background health check +// that can revive a stopped workspace reverses deliberate stops. +func workspaceTunnelHealth( + client client2.BaseWorkspaceClient, +) func(ctx context.Context) error { + return func(ctx context.Context) error { + status, err := client.Status(ctx, client2.StatusOptions{ContainerStatus: true}) + if err != nil { + return fmt.Errorf("workspace status: %w", err) + } + if status != client2.StatusRunning { + return fmt.Errorf("workspace is %q", status) + } + return nil + } +} + // startTunnel creates a local TCP tunnel that forwards connections to the // container SSH server. The returned cleanup function must be called when // the tunnel is no longer needed (typically via defer). @@ -30,8 +49,9 @@ func (cmd *UpCmd) startTunnel( } localTunnel, err := tunnel.NewLocalTunnel(ctx, tunnel.LocalTunnelOptions{ - BasePort: 10800, - DialFunc: dialer.Dial, + BasePort: 10800, + DialFunc: dialer.Dial, + HealthCheckFunc: workspaceTunnelHealth(client), }) if err != nil { return 0, nil, fmt.Errorf("create local tunnel: %w", err) diff --git a/cmd/workspace/up/tunnel_test.go b/cmd/workspace/up/tunnel_test.go new file mode 100644 index 0000000000..42ad6d3b65 --- /dev/null +++ b/cmd/workspace/up/tunnel_test.go @@ -0,0 +1,53 @@ +package up + +import ( + "context" + "errors" + "testing" + + client2 "github.com/devsy-org/devsy/pkg/client" +) + +// statusOnlyClient answers Status and panics on any other method through the +// embedded nil interface, so a health probe that reaches for a mutating path +// fails the test. +type statusOnlyClient struct { + client2.BaseWorkspaceClient + status client2.Status + err error +} + +func (c *statusOnlyClient) Status( + context.Context, + client2.StatusOptions, +) (client2.Status, error) { + return c.status, c.err +} + +func TestWorkspaceTunnelHealthHealthyOnlyWhenRunning(t *testing.T) { + for _, tc := range []struct { + name string + status client2.Status + err error + healthy bool + }{ + {"running", client2.StatusRunning, nil, true}, + {"stopped", client2.StatusStopped, nil, false}, + {"not found", client2.StatusNotFound, nil, false}, + {"provisioning", client2.StatusProvisioning, nil, false}, + {"failed", client2.StatusFailed, nil, false}, + {"busy", client2.StatusBusy, nil, false}, + {"status error", "", errors.New("daemon unreachable"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + health := workspaceTunnelHealth(&statusOnlyClient{status: tc.status, err: tc.err}) + err := health(t.Context()) + if tc.healthy && err != nil { + t.Errorf("health = %v, want nil", err) + } + if !tc.healthy && err == nil { + t.Error("health = nil, want failure") + } + }) + } +} diff --git a/pkg/tunnel/local_listener.go b/pkg/tunnel/local_listener.go index 410d9ca291..c59bd68e7b 100644 --- a/pkg/tunnel/local_listener.go +++ b/pkg/tunnel/local_listener.go @@ -20,6 +20,7 @@ type LocalTunnel struct { cancel context.CancelFunc wg sync.WaitGroup dialFunc DialFunc + healthCheckFunc func(ctx context.Context) error healthCheckInterval time.Duration } @@ -31,8 +32,14 @@ type DialFunc func(ctx context.Context) (io.ReadWriteCloser, error) type LocalTunnelOptions struct { // BasePort is the starting port to search from (default: 10800) BasePort int - // DialFunc creates a connection to the remote SSH endpoint + // DialFunc creates a connection to the remote SSH endpoint. It is the + // data path and runs only for real accepted local connections. DialFunc DialFunc + // HealthCheckFunc is the control-plane liveness probe. It must be + // observational: it must never start, restart, or otherwise mutate the + // resource it probes. When nil, no periodic probe runs and the listener + // lifetime follows its context. + HealthCheckFunc func(ctx context.Context) error // HealthCheckInterval overrides the default health check interval (for testing). // If zero, defaults to healthCheckInterval (30s). HealthCheckInterval time.Duration @@ -76,8 +83,11 @@ func NewLocalTunnel(ctx context.Context, opts LocalTunnelOptions) (*LocalTunnel, _ = listener.Close() }() - t.wg.Add(1) - go t.healthCheck() + if opts.HealthCheckFunc != nil { + t.healthCheckFunc = opts.HealthCheckFunc + t.wg.Add(1) + go t.healthCheck() + } return t, nil } @@ -112,8 +122,7 @@ func (t *LocalTunnel) healthCheck() { case <-t.ctx.Done(): return case <-ticker.C: - conn, err := t.dialFunc(t.ctx) - if err != nil { + if err := t.healthCheckFunc(t.ctx); err != nil { failures++ log.Debugf( "tunnel health check failed (%d/%d): %v", @@ -130,7 +139,6 @@ func (t *LocalTunnel) healthCheck() { return } } else { - _ = conn.Close() failures = 0 } } diff --git a/pkg/tunnel/local_listener_test.go b/pkg/tunnel/local_listener_test.go index dddc3c9fcb..60158e6d3c 100644 --- a/pkg/tunnel/local_listener_test.go +++ b/pkg/tunnel/local_listener_test.go @@ -7,6 +7,7 @@ import ( "io" "net" "sync" + "sync/atomic" "syscall" "testing" "time" @@ -226,6 +227,9 @@ func TestLocalTunnel_HealthCheckShutdown(t *testing.T) { DialFunc: func(ctx context.Context) (io.ReadWriteCloser, error) { return nil, fmt.Errorf("workspace gone") }, + HealthCheckFunc: func(context.Context) error { + return fmt.Errorf("workspace gone") + }, HealthCheckInterval: 50 * time.Millisecond, }) if err != nil { @@ -236,3 +240,92 @@ func TestLocalTunnel_HealthCheckShutdown(t *testing.T) { // The health check should shut down the tunnel after 3 failures. waitForListenerClosed(t, tun.Addr(), 2*time.Second) } + +func TestLocalTunnel_HealthCheckUsesHealthFuncNotDialFunc(t *testing.T) { + ctx := t.Context() + + // DialFunc always succeeds: if the health loop still probed through the + // data path, failures would never accumulate and the tunnel would stay + // alive. Only a health loop driven by the failing HealthCheckFunc shuts + // the listener down. + var healthCalls int32 + tun, err := NewLocalTunnel(ctx, LocalTunnelOptions{ + BasePort: 18600, + DialFunc: func(ctx context.Context) (io.ReadWriteCloser, error) { + local, remote := net.Pipe() + _ = local.Close() + return remote, nil + }, + HealthCheckFunc: func(context.Context) error { + atomic.AddInt32(&healthCalls, 1) + return fmt.Errorf("workspace gone") + }, + HealthCheckInterval: 50 * time.Millisecond, + }) + if err != nil { + t.Fatalf("NewLocalTunnel: %v", err) + } + defer func() { _ = tun.Close() }() + + waitForListenerClosed(t, tun.Addr(), 2*time.Second) + if got := atomic.LoadInt32(&healthCalls); got < 3 { + t.Errorf("health probe invoked %d times, want at least 3", got) + } +} + +func TestLocalTunnel_AcceptedConnectionStillUsesDialFunc(t *testing.T) { + ctx := t.Context() + echoServer := newEchoServer(t) + + var dialCalls int32 + tun, err := NewLocalTunnel(ctx, LocalTunnelOptions{ + BasePort: 18700, + DialFunc: func(ctx context.Context) (io.ReadWriteCloser, error) { + atomic.AddInt32(&dialCalls, 1) + return net.Dial("tcp", echoServer.Addr().String()) + }, + HealthCheckFunc: func(context.Context) error { + return nil + }, + HealthCheckInterval: 50 * time.Millisecond, + }) + if err != nil { + t.Fatalf("NewLocalTunnel: %v", err) + } + defer func() { _ = tun.Close() }() + + msg := []byte("data plane") + if buf := sendAndReceive(t, tun.Addr(), msg); string(buf) != string(msg) { + t.Errorf("expected %q, got %q", msg, buf) + } + if got := atomic.LoadInt32(&dialCalls); got == 0 { + t.Error("accepted connection did not invoke DialFunc") + } +} + +func TestLocalTunnel_HealthyStatusKeepsTunnelAlive(t *testing.T) { + ctx := t.Context() + + tun, err := NewLocalTunnel(ctx, LocalTunnelOptions{ + BasePort: 18800, + DialFunc: func(ctx context.Context) (io.ReadWriteCloser, error) { + return nil, io.EOF + }, + HealthCheckFunc: func(context.Context) error { + return nil + }, + HealthCheckInterval: 20 * time.Millisecond, + }) + if err != nil { + t.Fatalf("NewLocalTunnel: %v", err) + } + defer func() { _ = tun.Close() }() + + // Well past the failure window, a healthy probe keeps the listener up. + time.Sleep(200 * time.Millisecond) + conn, err := net.DialTimeout("tcp", tun.Addr(), time.Second) + if err != nil { + t.Fatalf("listener closed despite healthy probes: %v", err) + } + _ = conn.Close() +} From 853f30ac977d9f7977bfb7e7a268e90d64219957 Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:19:55 -0700 Subject: [PATCH 05/49] feat(command): own Windows background process trees with Job Objects taskkill /T terminates a process tree by parent linkage, so a descendant whose intermediate parent already exited escapes cancellation and can keep owning transport resources. Background workers launched through StartBackground are now assigned to a Job Object named after the worker PID at launch; membership is inherited by descendants and survives orphaning, and Kill terminates the job before falling back to taskkill /T /F for workers launched before job ownership existed. Job assignment failure is non-fatal and only narrows termination back to the taskkill behavior. Refs #1278 --- pkg/command/background.go | 9 +++ pkg/command/job_windows.go | 84 ++++++++++++++++++++++++++++ pkg/command/process_supported.go | 6 ++ pkg/command/process_windows.go | 35 +++++++++--- pkg/command/process_windows_test.go | 86 ++++++++++++++++++++++++----- 5 files changed, 199 insertions(+), 21 deletions(-) create mode 100644 pkg/command/job_windows.go diff --git a/pkg/command/background.go b/pkg/command/background.go index 70f97098d1..ee1f046302 100644 --- a/pkg/command/background.go +++ b/pkg/command/background.go @@ -117,6 +117,15 @@ func startDetached(cmd *exec.Cmd, pidFile, streamsFile string) error { } closeFile(streamsF) + if err := ownProcessTree(cmd.Process.Pid); err != nil { + fmt.Fprintf( + os.Stderr, + "warning: process tree ownership unavailable for pid %d: %v\n", + cmd.Process.Pid, + err, + ) + } + if pidFile != "" { if err := os.WriteFile(pidFile, []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { _ = cmd.Process.Kill() diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go new file mode 100644 index 0000000000..e8f4e64d63 --- /dev/null +++ b/pkg/command/job_windows.go @@ -0,0 +1,84 @@ +//go:build windows + +package command + +import ( + "errors" + "fmt" + "strconv" + "unsafe" + + "golang.org/x/sys/windows" +) + +const jobObjectTerminateAccess = 0x0008 + +var procOpenJobObjectW = windows.NewLazySystemDLL("kernel32.dll").NewProc("OpenJobObjectW") + +// jobNameForPID names the Job Object owning the process tree rooted at pid, +// so cancellation can recover the job from the persisted PID alone. +func jobNameForPID(pid int) string { + return "devsy-worker-" + strconv.Itoa(pid) +} + +// ownProcessTree assigns the process to a Job Object named after its PID. +// Parent-based termination (taskkill /T) loses track of descendants whose +// intermediate parent already exited; job membership survives that, so tree +// teardown stays complete. The launcher closes its handle right away: the +// job lives as long as it has member processes, which is what keeps the +// detached worker alive after the launcher exits. Assignment failure is +// non-fatal; termination then falls back to taskkill /T /F. +func ownProcessTree(pid int) error { + name, err := windows.UTF16PtrFromString(jobNameForPID(pid)) + if err != nil { + return err + } + job, err := windows.CreateJobObject(nil, name) + if err != nil { + return fmt.Errorf("create job object: %w", err) + } + defer func() { _ = windows.CloseHandle(job) }() + + handle, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + false, + uint32(pid), + ) + if err != nil { + return fmt.Errorf("open process %d: %w", pid, err) + } + defer func() { _ = windows.CloseHandle(handle) }() + + if err := windows.AssignProcessToJobObject(job, handle); err != nil { + return fmt.Errorf("assign process %d to job: %w", pid, err) + } + return nil +} + +// terminateJob kills the process tree owned by the Job Object named for pid. +// The bool reports whether a job existed; without one the caller falls back +// to taskkill, e.g. for workers launched before job ownership existed. +func terminateJob(pid int) (bool, error) { + name, err := windows.UTF16PtrFromString(jobNameForPID(pid)) + if err != nil { + return false, err + } + handle, _, callErr := procOpenJobObjectW.Call( + uintptr(jobObjectTerminateAccess), + 0, + uintptr(unsafe.Pointer(name)), + ) + if handle == 0 { + if errors.Is(callErr, windows.ERROR_FILE_NOT_FOUND) { + return false, nil + } + return false, fmt.Errorf("open job for pid %d: %w", pid, callErr) + } + job := windows.Handle(handle) + defer func() { _ = windows.CloseHandle(job) }() + + if err := windows.TerminateJobObject(job, 1); err != nil { + return false, fmt.Errorf("terminate job for pid %d: %w", pid, err) + } + return true, nil +} diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index 4ec57a98a2..eba3bce5bd 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -48,3 +48,9 @@ func kill(pid string) error { } return nil } + +// ownProcessTree is a no-op on Unix: process-group termination via the +// worker's PID already covers the tree, and Job Objects are Windows-only. +func ownProcessTree(pid int) error { + return nil +} diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index 87fdd32336..74e36bc022 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -7,6 +7,7 @@ import ( "fmt" "os/exec" "strconv" + "time" "golang.org/x/sys/windows" ) @@ -56,6 +57,13 @@ func kill(pid string) error { return nil } + // Job Object termination tears down the whole tree, including orphans + // that parent-based termination cannot reach. Workers launched before + // job ownership existed fall through to taskkill. + if terminated, jobErr := terminateJob(parsed); jobErr == nil && terminated { + return verifyTerminated(parsed) + } + // /T takes down the worker's descendants as well: a detached up worker // can spawn children (e.g. devsy workspace ssh --stdio) that keep owning // transport resources. /F stands in for SIGKILL; Windows has no SIGTERM @@ -76,14 +84,27 @@ func kill(pid string) error { ) } - stillRunning, err := isRunning(pid) - if err != nil { - return err - } - if stillRunning { - return fmt.Errorf("taskkill /PID %d /T /F reported success but the process is still running", parsed) + return verifyTerminated(parsed) +} + +// verifyTerminated confirms the process is gone, allowing a brief window for +// the exit to become visible after the termination call reported success. +func verifyTerminated(parsed int) error { + pid := strconv.Itoa(parsed) + deadline := time.Now().Add(5 * time.Second) + for { + stillRunning, err := isRunning(pid) + if err != nil { + return err + } + if !stillRunning { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("process %d is still running after termination", parsed) + } + time.Sleep(100 * time.Millisecond) } - return nil } func parsePID(pid string) (int, error) { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 40494fc4fe..9659709c4c 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -13,8 +13,11 @@ import ( ) const ( - helperEnvMarker = "DEVSY_TEST_HELPER_PROCESS" - helperEnvPIDFile = "DEVSY_TEST_HELPER_PIDFILE" + helperEnvMarker = "DEVSY_TEST_HELPER_PROCESS" + helperEnvPIDFile = "DEVSY_TEST_HELPER_PIDFILE" + helperEnvChildPIDFile = "DEVSY_TEST_HELPER_CHILD_PIDFILE" + helperEnvWaitFile = "DEVSY_TEST_HELPER_WAIT_FILE" + helperEnvExitAfterSpawn = "DEVSY_TEST_HELPER_EXIT_AFTER_SPAWN" ) // TestHelperProcess is not a test; every helper process re-executes the test @@ -24,9 +27,23 @@ func TestHelperProcess(t *testing.T) { if os.Getenv(helperEnvMarker) != "1" { return } + if waitFile := os.Getenv(helperEnvWaitFile); waitFile != "" { + for { + if _, err := os.Stat(waitFile); err == nil { + break + } + time.Sleep(50 * time.Millisecond) + } + } if pidFile := os.Getenv(helperEnvPIDFile); pidFile != "" { child := exec.Command(os.Args[0], "-test.run", "TestHelperProcess") - child.Env = helperEnv() + child.Env = helperEnv(helperEnvMarker + "=1") + if grandchildPIDFile := os.Getenv(helperEnvChildPIDFile); grandchildPIDFile != "" { + child.Env = append(child.Env, + helperEnvPIDFile+"="+grandchildPIDFile, + helperEnvExitAfterSpawn+"=1", + ) + } if err := child.Start(); err != nil { os.Exit(1) } @@ -37,6 +54,9 @@ func TestHelperProcess(t *testing.T) { ); err != nil { os.Exit(1) } + if os.Getenv(helperEnvExitAfterSpawn) == "1" { + os.Exit(0) + } } for { time.Sleep(time.Hour) @@ -107,25 +127,28 @@ func TestIsRunningDetectsLiveAndExitedProcess(t *testing.T) { assertNotRunningEventually(t, cmd.Process.Pid) } -func TestKillTerminatesProcessTree(t *testing.T) { - pidFile := filepath.Join(t.TempDir(), "child.pid") - parent := startHelper(t, helperEnvPIDFile+"="+pidFile) - - var childPID int +func waitForPIDFile(t *testing.T, pidFile string) int { + t.Helper() deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { data, err := os.ReadFile(pidFile) // #nosec G304 -- test-controlled path if err == nil { - childPID, err = strconv.Atoi(strings.TrimSpace(string(data))) - if err == nil { - break + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err == nil && pid > 0 { + return pid } } time.Sleep(100 * time.Millisecond) } - if childPID == 0 { - t.Fatal("helper parent never reported its child PID") - } + t.Fatalf("no PID reported in %s", pidFile) + return 0 +} + +func TestKillTerminatesProcessTree(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "child.pid") + parent := startHelper(t, helperEnvPIDFile+"="+pidFile) + + childPID := waitForPIDFile(t, pidFile) assertRunning(t, parent.Process.Pid) assertRunning(t, childPID) @@ -173,3 +196,38 @@ func TestInvalidPIDReturnsError(t *testing.T) { } } } + +func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { + dir := t.TempDir() + waitFile := filepath.Join(dir, "go") + childPIDFile := filepath.Join(dir, "child.pid") + grandchildPIDFile := filepath.Join(dir, "grandchild.pid") + + // The parent waits so the job can own it before it spawns anything; the + // child then spawns the grandchild and exits, orphaning the grandchild + // from the perspective of parent-based termination. + parent := startHelper(t, + helperEnvWaitFile+"="+waitFile, + helperEnvPIDFile+"="+childPIDFile, + helperEnvChildPIDFile+"="+grandchildPIDFile, + ) + if err := ownProcessTree(parent.Process.Pid); err != nil { + t.Skipf("job assignment unavailable in this environment: %v", err) + } + if err := os.WriteFile(waitFile, []byte("go"), 0o600); err != nil { + t.Fatalf("signal helper: %v", err) + } + + childPID := waitForPIDFile(t, childPIDFile) + grandchildPID := waitForPIDFile(t, grandchildPIDFile) + assertNotRunningEventually(t, childPID) + assertRunning(t, parent.Process.Pid) + assertRunning(t, grandchildPID) + + if err := Kill(strconv.Itoa(parent.Process.Pid)); err != nil { + t.Fatalf("Kill(parent): %v", err) + } + + assertNotRunningEventually(t, parent.Process.Pid) + assertNotRunningEventually(t, grandchildPID) +} From 1d618ed2367ba3e0d05a7b500bc3b300a614d3af Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:34:16 -0700 Subject: [PATCH 06/49] test(e2e): cover Windows task cancellation and tunnel lifecycle Run the ssh-tunnel-mode suite through detached up with product-API cleanup (task cancel) instead of Unix SIGINT, enabling it on Windows with the podman provider. Add a task-cancel e2e suite asserting cancel does not panic, records the canceled state, reaps the worker process, and that stop quiesces a persisted detached task while the tunnel health check never restarts a stopped workspace. Wire both suites into pr-ci for windows-latest, and task-cancel for ubuntu-latest. --- .github/workflows/pr-ci.yml | 17 ++++ e2e/framework/command.go | 12 +++ e2e/tests/ssh/detached_task.go | 141 ++++++++++++++++++++++++++ e2e/tests/ssh/ssh_tunnel_mode.go | 169 +++++-------------------------- e2e/tests/ssh/task_cancel.go | 119 ++++++++++++++++++++++ 5 files changed, 314 insertions(+), 144 deletions(-) create mode 100644 e2e/tests/ssh/detached_task.go create mode 100644 e2e/tests/ssh/task_cancel.go diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index c690f16b15..6541c572e0 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -302,6 +302,12 @@ jobs: install-kind: false requires-secret: false + - label: task-cancel + runner: ubuntu-latest + free-disk-space: false + install-kind: false + requires-secret: false + - label: ssh-credentials-server-race runner: ubuntu-latest free-disk-space: false @@ -592,6 +598,17 @@ jobs: free-disk-space: false install-kind: true requires-secret: false + - label: ssh-tunnel-mode + runner: windows-latest + free-disk-space: false + install-kind: true + requires-secret: false + + - label: task-cancel + runner: windows-latest + free-disk-space: false + install-kind: true + requires-secret: false runs-on: ${{ matrix.runner }} diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 14fe296ad6..2912067c6c 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -32,6 +32,7 @@ const ( cmdList = "list" cmdGet = "get" cmdDelete = "delete" + cmdCancel = "cancel" cmdSSH = "ssh" cmdProvider = "provider" cmdWorkspace = "workspace" @@ -478,6 +479,17 @@ func (f *Framework) DevsyWorkspaceStop(ctx context.Context, extraArgs ...string) return f.ExecCommandStdout(ctx, baseArgs) } +// DevsyWorkspaceTaskCancel cancels a detached workspace task. +func (f *Framework) DevsyWorkspaceTaskCancel(ctx context.Context, taskID string) error { + return f.ExecCommandStdout(ctx, []string{cmdWorkspace, "task", cmdCancel, taskID}) +} + +// DevsyWorkspaceTaskList returns the raw `workspace task list` JSON states. +func (f *Framework) DevsyWorkspaceTaskList(ctx context.Context) (string, error) { + return f.ExecCommandOutput(ctx, + []string{cmdWorkspace, "task", cmdList, flagResultFormat, formatJSON}) +} + func (f *Framework) DevsyWorkspaceDelete( ctx context.Context, workspace string, diff --git a/e2e/tests/ssh/detached_task.go b/e2e/tests/ssh/detached_task.go new file mode 100644 index 0000000000..58807c4b8f --- /dev/null +++ b/e2e/tests/ssh/detached_task.go @@ -0,0 +1,141 @@ +package ssh + +import ( + "context" + "encoding/json" + "fmt" + "runtime" + "strings" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/onsi/gomega" +) + +// detachedUpState mirrors the `workspace task list` JSON fields the detached +// up specs poll on. +type detachedUpState struct { + ID string `json:"id"` + Status string `json:"status"` + Phase string `json:"phase"` + ErrorCode string `json:"errorCode"` + Error string `json:"error"` + PID int `json:"pid"` +} + +// setupTunnelProvider registers the docker provider; Windows runs use the +// podman runtime installed by CI. +func setupTunnelProvider(ctx context.Context, initialDir string) *framework.Framework { + binDir := initialDir + "/bin" + if runtime.GOOS == osWindows { + f, err := framework.SetupDockerProvider(binDir, "podman") + framework.ExpectNoError(err) + return f + } + f := framework.NewDefaultFramework(binDir) + _ = f.DevsyProviderAdd(ctx, "docker") + framework.ExpectNoError(f.DevsyProviderUse(ctx, "docker")) + return f +} + +// startDetachedTunnelUp submits `workspace up --detach --ssh-tunnel` and +// returns the task ID from the envelope on stdout. +func startDetachedTunnelUp( + ctx context.Context, + f *framework.Framework, + workspace string, + extraArgs ...string, +) (string, error) { + args := []string{ + cmdWorkspace, "up", + names.Flag(names.Debug), + names.Flag(names.IDE), "none", + names.Flag(names.SSHTunnel), + names.Flag(names.Detach), + names.Flag(names.ResultFormat), "json", + } + args = append(args, extraArgs...) + args = append(args, workspace) + + out, err := f.ExecCommandOutput(ctx, args) + if err != nil { + return "", err + } + var env struct { + Kind string `json:"kind"` + ID string `json:"id"` + } + line, _, _ := strings.Cut(strings.TrimSpace(out), "\n") + if err := json.Unmarshal([]byte(line), &env); err != nil { + return "", fmt.Errorf("parse task envelope %q: %w", out, err) + } + if env.Kind != "task" || env.ID == "" { + return "", fmt.Errorf("unexpected task envelope %q", line) + } + return env.ID, nil +} + +func findDetachedUpTask( + ctx context.Context, + f *framework.Framework, + taskID string, +) (detachedUpState, error) { + out, err := f.DevsyWorkspaceTaskList(ctx) + if err != nil { + return detachedUpState{}, err + } + var states []detachedUpState + if err := json.Unmarshal([]byte(out), &states); err != nil { + return detachedUpState{}, fmt.Errorf("parse task list: %w", err) + } + for _, s := range states { + if s.ID == taskID { + return s, nil + } + } + return detachedUpState{}, fmt.Errorf("task %s not in task list", taskID) +} + +// waitDetachedTunnelReady polls the task until the up pipeline reports the +// ready phase, which it reaches only with the tunnel active. +func waitDetachedTunnelReady( + ctx context.Context, + f *framework.Framework, + taskID string, +) detachedUpState { + var state detachedUpState + gomega.Eventually(func() (string, error) { + var err error + state, err = findDetachedUpTask(ctx, f, taskID) + if err != nil { + return "", err + } + if state.Status == "failed" { + return "", gomega.StopTrying("detached up task failed: " + state.Error) + } + return state.Phase, nil + }).WithContext(ctx).WithTimeout(tunnelActiveTimeout).WithPolling(2 * time.Second). + Should(gomega.Equal("ready")) + return state +} + +// waitDetachedTaskCanceled polls until the task records the canceled error +// code and returns the final state. +func waitDetachedTaskCanceled( + ctx context.Context, + f *framework.Framework, + taskID string, +) detachedUpState { + var state detachedUpState + gomega.Eventually(func() string { + var err error + state, err = findDetachedUpTask(ctx, f, taskID) + if err != nil { + return "" + } + return state.ErrorCode + }).WithContext(ctx).WithTimeout(30 * time.Second).WithPolling(time.Second). + Should(gomega.Equal("canceled")) + return state +} diff --git a/e2e/tests/ssh/ssh_tunnel_mode.go b/e2e/tests/ssh/ssh_tunnel_mode.go index 7692a7b9f7..4203ba854a 100644 --- a/e2e/tests/ssh/ssh_tunnel_mode.go +++ b/e2e/tests/ssh/ssh_tunnel_mode.go @@ -1,112 +1,20 @@ package ssh import ( - "bytes" "context" - "fmt" "net" "os" - "os/exec" "path/filepath" - "runtime" "strings" - "sync" - "syscall" "time" "github.com/devsy-org/devsy/e2e/framework" - "github.com/devsy-org/devsy/pkg/flags/names" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) -const tunnelActiveMarker = "waiting for shutdown signal" - const tunnelActiveTimeout = 4 * time.Minute -type safeBuffer struct { - mu sync.Mutex - buf bytes.Buffer -} - -func (b *safeBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.Write(p) -} - -func (b *safeBuffer) String() string { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.String() -} - -type tunnelUpProcess struct { - cmd *exec.Cmd - output *safeBuffer - exited chan struct{} - exitErr error -} - -func startTunnelUp( - f *framework.Framework, workspace string, extraArgs ...string, -) (*tunnelUpProcess, error) { - args := []string{ - cmdWorkspace, "up", - names.Flag(names.Debug), - names.Flag(names.IDE), "none", - names.Flag(names.SSHTunnel), - } - args = append(args, extraArgs...) - args = append(args, workspace) - - // #nosec G204 -- test binary with controlled arguments - cmd := exec.Command(filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) - out := &safeBuffer{} - cmd.Stdout = out - cmd.Stderr = out - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("start devsy up --ssh-tunnel: %w", err) - } - - p := &tunnelUpProcess{cmd: cmd, output: out, exited: make(chan struct{})} - go func() { - p.exitErr = cmd.Wait() - close(p.exited) - }() - return p, nil -} - -func (p *tunnelUpProcess) waitUntilActive(ctx context.Context) { - gomega.Eventually(func() (string, error) { - out := p.output.String() - select { - case <-p.exited: - return out, gomega.StopTrying( - "devsy up exited before the tunnel became active", - ).Wrap(p.exitErr) - default: - return out, nil - } - }).WithContext(ctx).WithTimeout(tunnelActiveTimeout).WithPolling(100 * time.Millisecond). - Should(gomega.ContainSubstring(tunnelActiveMarker)) -} - -func (p *tunnelUpProcess) stop() { - select { - case <-p.exited: - return - default: - } - _ = p.cmd.Process.Signal(syscall.SIGINT) - select { - case <-p.exited: - case <-time.After(15 * time.Second): - _ = p.cmd.Process.Kill() - <-p.exited - } -} - var _ = ginkgo.Describe( "devsy ssh tunnel mode", ginkgo.Label("ssh-tunnel-mode"), @@ -123,28 +31,23 @@ var _ = ginkgo.Describe( ginkgo.It("should start workspace with --ssh-tunnel and SSH into it", ginkgo.SpecTimeout(framework.TimeoutModerate()), func(ctx context.Context) { - if runtime.GOOS == osWindows { - ginkgo.Skip("skipping on windows") - } - tempDir, err := framework.CopyToTempDir("tests/ssh/testdata/local-test") framework.ExpectNoError(err) - f := framework.NewDefaultFramework(initialDir + "/bin") - _ = f.DevsyProviderAdd(ctx, "docker") - err = f.DevsyProviderUse(ctx, "docker") - framework.ExpectNoError(err) + f := setupTunnelProvider(ctx, initialDir) ginkgo.DeferCleanup(func(cleanupCtx context.Context) { _ = f.DevsyWorkspaceDelete(cleanupCtx, tempDir) framework.CleanupTempDir(initialDir, tempDir) }) - proc, err := startTunnelUp(f, tempDir) + taskID, err := startDetachedTunnelUp(ctx, f, tempDir) framework.ExpectNoError(err) - ginkgo.DeferCleanup(proc.stop) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = f.DevsyWorkspaceTaskCancel(cleanupCtx, taskID) + }) - proc.waitUntilActive(ctx) + waitDetachedTunnelReady(ctx, f, taskID) devsySSHCtx, cancelSSH := context.WithDeadline(ctx, time.Now().Add(20*time.Second)) defer cancelSSH() @@ -156,31 +59,26 @@ var _ = ginkgo.Describe( ginkgo.It("should write SSH config with Hostname and Port instead of ProxyCommand", ginkgo.SpecTimeout(framework.TimeoutModerate()), func(ctx context.Context) { - if runtime.GOOS == osWindows { - ginkgo.Skip("skipping on windows") - } - tempDir, err := framework.CopyToTempDir("tests/ssh/testdata/local-test") framework.ExpectNoError(err) sshConfigDir := ginkgo.GinkgoT().TempDir() sshConfigPath := filepath.Join(sshConfigDir, "config") - f := framework.NewDefaultFramework(initialDir + "/bin") - _ = f.DevsyProviderAdd(ctx, "docker") - err = f.DevsyProviderUse(ctx, "docker") - framework.ExpectNoError(err) + f := setupTunnelProvider(ctx, initialDir) ginkgo.DeferCleanup(func(cleanupCtx context.Context) { _ = f.DevsyWorkspaceDelete(cleanupCtx, tempDir) framework.CleanupTempDir(initialDir, tempDir) }) - proc, err := startTunnelUp(f, tempDir, "--ssh-config", sshConfigPath) + taskID, err := startDetachedTunnelUp(ctx, f, tempDir, "--ssh-config", sshConfigPath) framework.ExpectNoError(err) - ginkgo.DeferCleanup(proc.stop) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = f.DevsyWorkspaceTaskCancel(cleanupCtx, taskID) + }) - proc.waitUntilActive(ctx) + waitDetachedTunnelReady(ctx, f, taskID) configBytes, err := os.ReadFile(filepath.Clean(sshConfigPath)) framework.ExpectNoError(err) @@ -204,31 +102,26 @@ var _ = ginkgo.Describe( ginkgo.It("should establish a working local TCP tunnel listener", ginkgo.SpecTimeout(framework.TimeoutModerate()), func(ctx context.Context) { - if runtime.GOOS == osWindows { - ginkgo.Skip("skipping on windows") - } - tempDir, err := framework.CopyToTempDir("tests/ssh/testdata/local-test") framework.ExpectNoError(err) sshConfigDir := ginkgo.GinkgoT().TempDir() sshConfigPath := filepath.Join(sshConfigDir, "config") - f := framework.NewDefaultFramework(initialDir + "/bin") - _ = f.DevsyProviderAdd(ctx, "docker") - err = f.DevsyProviderUse(ctx, "docker") - framework.ExpectNoError(err) + f := setupTunnelProvider(ctx, initialDir) ginkgo.DeferCleanup(func(cleanupCtx context.Context) { _ = f.DevsyWorkspaceDelete(cleanupCtx, tempDir) framework.CleanupTempDir(initialDir, tempDir) }) - proc, err := startTunnelUp(f, tempDir, "--ssh-config", sshConfigPath) + taskID, err := startDetachedTunnelUp(ctx, f, tempDir, "--ssh-config", sshConfigPath) framework.ExpectNoError(err) - ginkgo.DeferCleanup(proc.stop) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = f.DevsyWorkspaceTaskCancel(cleanupCtx, taskID) + }) - proc.waitUntilActive(ctx) + waitDetachedTunnelReady(ctx, f, taskID) configBytes, err := os.ReadFile(filepath.Clean(sshConfigPath)) framework.ExpectNoError(err) @@ -256,28 +149,23 @@ var _ = ginkgo.Describe( ginkgo.It("should handle multiple sequential SSH commands via tunnel", ginkgo.SpecTimeout(framework.TimeoutModerate()), func(ctx context.Context) { - if runtime.GOOS == osWindows { - ginkgo.Skip("skipping on windows") - } - tempDir, err := framework.CopyToTempDir("tests/ssh/testdata/local-test") framework.ExpectNoError(err) - f := framework.NewDefaultFramework(initialDir + "/bin") - _ = f.DevsyProviderAdd(ctx, "docker") - err = f.DevsyProviderUse(ctx, "docker") - framework.ExpectNoError(err) + f := setupTunnelProvider(ctx, initialDir) ginkgo.DeferCleanup(func(cleanupCtx context.Context) { _ = f.DevsyWorkspaceDelete(cleanupCtx, tempDir) framework.CleanupTempDir(initialDir, tempDir) }) - proc, err := startTunnelUp(f, tempDir) + taskID, err := startDetachedTunnelUp(ctx, f, tempDir) framework.ExpectNoError(err) - ginkgo.DeferCleanup(proc.stop) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = f.DevsyWorkspaceTaskCancel(cleanupCtx, taskID) + }) - proc.waitUntilActive(ctx) + waitDetachedTunnelReady(ctx, f, taskID) for i := range 3 { sshCtx, cancelSSH := context.WithDeadline(ctx, time.Now().Add(20*time.Second)) @@ -299,20 +187,13 @@ var _ = ginkgo.Describe( ginkgo.It("should fall back to ProxyCommand when tunnel mode is not enabled", ginkgo.SpecTimeout(framework.TimeoutModerate()), func(ctx context.Context) { - if runtime.GOOS == osWindows { - ginkgo.Skip("skipping on windows") - } - tempDir, err := framework.CopyToTempDir("tests/ssh/testdata/local-test") framework.ExpectNoError(err) sshConfigDir := ginkgo.GinkgoT().TempDir() sshConfigPath := filepath.Join(sshConfigDir, "config") - f := framework.NewDefaultFramework(initialDir + "/bin") - _ = f.DevsyProviderAdd(ctx, "docker") - err = f.DevsyProviderUse(ctx, "docker") - framework.ExpectNoError(err) + f := setupTunnelProvider(ctx, initialDir) ginkgo.DeferCleanup(func(cleanupCtx context.Context) { _ = f.DevsyWorkspaceDelete(cleanupCtx, tempDir) diff --git a/e2e/tests/ssh/task_cancel.go b/e2e/tests/ssh/task_cancel.go new file mode 100644 index 0000000000..2c3abfeb4b --- /dev/null +++ b/e2e/tests/ssh/task_cancel.go @@ -0,0 +1,119 @@ +package ssh + +import ( + "context" + "os" + "strconv" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/command" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +// stoppedHoldWindow outlasts the tunnel health-check interval and failure +// threshold (30s x 3), so a workspace still Stopped after the window was not +// restarted by the health loop. +const stoppedHoldWindow = 100 * time.Second + +var _ = ginkgo.Describe( + "devsy workspace task cancel", + ginkgo.Label("task-cancel"), + ginkgo.Ordered, + func() { + var initialDir string + + ginkgo.BeforeEach(func() { + var err error + initialDir, err = os.Getwd() + framework.ExpectNoError(err) + }) + + ginkgo.It("should cancel a detached tunnel task and keep the workspace stopped", + ginkgo.SpecTimeout(framework.TimeoutLong()), + func(ctx context.Context) { + tempDir, err := framework.CopyToTempDir("tests/ssh/testdata/local-test") + framework.ExpectNoError(err) + + f := setupTunnelProvider(ctx, initialDir) + + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = f.DevsyWorkspaceDelete(cleanupCtx, tempDir) + framework.CleanupTempDir(initialDir, tempDir) + }) + + taskID, err := startDetachedTunnelUp(ctx, f, tempDir) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = f.DevsyWorkspaceTaskCancel(cleanupCtx, taskID) + }) + + active := waitDetachedTunnelReady(ctx, f, taskID) + gomega.Expect(active.PID).NotTo(gomega.BeZero(), + "detached up worker should publish its pid") + + // Cancellation goes through the product task API; on Windows + // this used to panic with "unsupported". + framework.ExpectNoError(f.DevsyWorkspaceTaskCancel(ctx, taskID)) + + canceled := waitDetachedTaskCanceled(ctx, f, taskID) + gomega.Expect(canceled.Error).To(gomega.Equal("canceled")) + + gomega.Eventually(func() bool { + running, err := command.IsRunning(strconv.Itoa(active.PID)) + return err == nil && !running + }).WithContext(ctx).WithTimeout(30*time.Second).WithPolling(time.Second). + Should(gomega.BeTrue(), + "detached worker process should be gone after cancel") + + framework.ExpectNoError(f.DevsyStop(ctx, tempDir)) + + expectWorkspaceStaysStopped(ctx, f, tempDir) + }, + ) + + ginkgo.It("should quiesce a persisted detached task when stop runs without a task ID", + ginkgo.SpecTimeout(framework.TimeoutLong()), + func(ctx context.Context) { + tempDir, err := framework.CopyToTempDir("tests/ssh/testdata/local-test") + framework.ExpectNoError(err) + + f := setupTunnelProvider(ctx, initialDir) + + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = f.DevsyWorkspaceDelete(cleanupCtx, tempDir) + framework.CleanupTempDir(initialDir, tempDir) + }) + + taskID, err := startDetachedTunnelUp(ctx, f, tempDir) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + _ = f.DevsyWorkspaceTaskCancel(cleanupCtx, taskID) + }) + + waitDetachedTunnelReady(ctx, f, taskID) + + // Stop discovers the persisted task on its own and cancels it. + framework.ExpectNoError(f.DevsyStop(ctx, tempDir)) + + canceled := waitDetachedTaskCanceled(ctx, f, taskID) + gomega.Expect(canceled.Error).To(gomega.Equal("canceled")) + + expectWorkspaceStaysStopped(ctx, f, tempDir) + }, + ) + }, +) + +func expectWorkspaceStaysStopped(ctx context.Context, f *framework.Framework, workspace string) { + gomega.Consistently(func() string { + status, err := f.DevsyStatus(ctx, workspace) + if err != nil { + return "status error: " + err.Error() + } + return status.State + }).WithContext(ctx).WithTimeout(stoppedHoldWindow).WithPolling(10 * time.Second). + Should(gomega.Equal(client.StatusStopped)) +} From 7de8357af88d5249535430cca54ffdfd1e36d8e1 Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:34:16 -0700 Subject: [PATCH 07/49] docs: note background up task reaping on stop and delete --- .../stop-and-delete-a-workspace.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/stop-and-delete-a-workspace.mdx b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/stop-and-delete-a-workspace.mdx index 904ad87298..b362a2605f 100644 --- a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/stop-and-delete-a-workspace.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/stop-and-delete-a-workspace.mdx @@ -32,6 +32,12 @@ Some providers allow automatic stop of a workspace, usually to save costs when a For example, the Google Cloud provider will shutdown the virtual machine where the workspace is running after it's unused for 10 minutes, by default. This means you need to start the workspace after it has been stopped when you want to reconnect. +### Background up tasks + +Stopping or deleting a workspace first cancels any active detached `devsy workspace up --detach` task for that workspace, so a background up cannot keep recreating or restarting a workspace you asked to stop. +Detached task cancellation is supported on all platforms, including Windows: `devsy workspace task cancel ` terminates the task's process tree and records the task as canceled. +While an SSH tunnel is attached, its health checks observe the workspace but never start a stopped one. + ## Delete a Workspace Deleting a workspace means to erase all state of an existing workspace and remove all traces of it within Devsy. From eae433692f2d45aabd8ce2a868723df8a7405b9a Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 23:23:48 -0700 Subject: [PATCH 08/49] style(task,tunnel): satisfy repo lint gates Address golangci-lint findings in the new task cancellation and tunnel lifecycle code: method ordering, test cyclomatic complexity, repeated string constants, atomic.Int32, and formatter output. --- pkg/task/store.go | 76 +++++++++++++++---------------- pkg/task/store_test.go | 70 ++++++++++++++++------------ pkg/task/task_test.go | 66 +++++++++++++-------------- pkg/tunnel/local_listener_test.go | 24 +++++----- 4 files changed, 122 insertions(+), 114 deletions(-) diff --git a/pkg/task/store.go b/pkg/task/store.go index 7f23ae3648..6dc0348885 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -170,6 +170,44 @@ func (s *Store) List() ([]*State, error) { return states, nil } +// ActiveForWorkspace returns the non-terminal tasks labeled with the exact +// workspace ID, newest first, reconciling stale workers first so an +// abandoned task does not read as live. A non-empty command restricts the +// match to that command. The store only answers which persisted operations +// are still active; callers decide what to do with them. +func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error) { + states, err := s.List() + if err != nil { + return nil, err + } + + active := make([]*State, 0, len(states)) + for _, state := range states { + if state.WorkspaceID != workspaceID { + continue + } + if command != "" && state.Command != command { + continue + } + state = s.Reconcile(state) + if state.Status.Terminal() { + continue + } + active = append(active, state) + } + return active, nil +} + +// SetKillProcessForTest replaces this store's process termination hook, so a +// test can drive cancellation outcomes deterministically without real PIDs. +// Production code must never call it. +// +// Not in export_test.go: other packages' tests need it, and a _test.go file is +// only compiled into its own package's test binary. +func (s *Store) SetKillProcessForTest(fn func(string) error) { + s.killProcess = fn +} + // failAbandoned records ErrAbandoned for a task whose worker is confirmed gone. // Must be called with the worker lock held. func (s *Store) failAbandoned(state *State) *State { @@ -323,41 +361,3 @@ func syncDir(dir string) { defer func() { _ = d.Close() }() _ = d.Sync() } - -// SetKillProcessForTest replaces this store's process termination hook, so a -// test can drive cancellation outcomes deterministically without real PIDs. -// Production code must never call it. -// -// Not in export_test.go: other packages' tests need it, and a _test.go file is -// only compiled into its own package's test binary. -func (s *Store) SetKillProcessForTest(fn func(string) error) { - s.killProcess = fn -} - -// ActiveForWorkspace returns the non-terminal tasks labeled with the exact -// workspace ID, newest first, reconciling stale workers first so an -// abandoned task does not read as live. A non-empty command restricts the -// match to that command. The store only answers which persisted operations -// are still active; callers decide what to do with them. -func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error) { - states, err := s.List() - if err != nil { - return nil, err - } - - active := make([]*State, 0, len(states)) - for _, state := range states { - if state.WorkspaceID != workspaceID { - continue - } - if command != "" && state.Command != command { - continue - } - state = s.Reconcile(state) - if state.Status.Terminal() { - continue - } - active = append(active, state) - } - return active, nil -} diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index 358e9388b1..606a23d91f 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -5,6 +5,8 @@ import ( "time" ) +const workspaceOne = "ws-1" + func createTaskWith(t *testing.T, store *Store, opts CreateOptions) *Task { t.Helper() tk, err := store.Create(opts) @@ -14,54 +16,62 @@ func createTaskWith(t *testing.T, store *Store, opts CreateOptions) *Task { return tk } -func TestActiveForWorkspaceFiltersAndOrders(t *testing.T) { +func queryActive(t *testing.T, store *Store, workspaceID, command string) []*State { + t.Helper() + active, err := store.ActiveForWorkspace(workspaceID, command) + if err != nil { + t.Fatalf("ActiveForWorkspace: %v", err) + } + return active +} + +func TestActiveForWorkspaceFilters(t *testing.T) { store := newTestStore(t) - older := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) - if err := store.update(older.ID(), func(s *State) { - s.StartedAt = s.StartedAt.Add(-time.Hour) - }); err != nil { - t.Fatalf("backdate: %v", err) - } - newer := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) - otherCommand := createTaskWith(t, store, CreateOptions{Command: "build", WorkspaceID: "ws-1"}) + up := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) + _ = createTaskWith(t, store, CreateOptions{Command: "build", WorkspaceID: workspaceOne}) otherWorkspace := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-2"}) - terminal := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) + terminal := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) if err := terminal.Succeed(nil); err != nil { t.Fatalf("Succeed: %v", err) } - active, err := store.ActiveForWorkspace("ws-1", "up") - if err != nil { - t.Fatalf("ActiveForWorkspace: %v", err) + // The command filter keeps only live up tasks for the workspace. + active := queryActive(t, store, workspaceOne, "up") + if len(active) != 1 || active[0].ID != up.ID() { + t.Errorf("command-filtered match = %+v, want only the live up task", active) } - if len(active) != 2 { - t.Fatalf("got %d active tasks, want 2: %+v", len(active), active) + if all := queryActive(t, store, workspaceOne, ""); len(all) != 2 { + t.Errorf("unfiltered match = %d, want 2", len(all)) } - if active[0].ID != newer.ID() || active[1].ID != older.ID() { - t.Errorf("unexpected order/contents: %s, %s", active[0].ID, active[1].ID) + if none := queryActive(t, store, otherWorkspace.ID(), "up"); len(none) != 0 { + t.Errorf("task id used as workspace id matched %d tasks, want 0", len(none)) } +} - all, err := store.ActiveForWorkspace("ws-1", "") - if err != nil { - t.Fatalf("ActiveForWorkspace: %v", err) - } - if len(all) != 3 { - t.Errorf("unfiltered match = %d, want 3 (includes %s)", len(all), otherCommand.ID()) +func TestActiveForWorkspaceOrdersNewestFirst(t *testing.T) { + store := newTestStore(t) + + older := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) + if err := store.update(older.ID(), func(s *State) { + s.StartedAt = s.StartedAt.Add(-time.Hour) + }); err != nil { + t.Fatalf("backdate: %v", err) } + newer := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) - none, err := store.ActiveForWorkspace(otherWorkspace.ID(), "up") + active, err := store.ActiveForWorkspace(workspaceOne, "up") if err != nil { t.Fatalf("ActiveForWorkspace: %v", err) } - if len(none) != 0 { - t.Errorf("task id used as workspace id matched %d tasks, want 0", len(none)) + if len(active) != 2 || active[0].ID != newer.ID() || active[1].ID != older.ID() { + t.Errorf("unexpected order/contents: %+v", active) } } func TestActiveForWorkspaceReconcilesAbandonedWorkers(t *testing.T) { store := newTestStore(t) - tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) + tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) // A worker that claimed then lost its lock died without recording a // result; the query must reconcile it instead of reporting it active. @@ -72,7 +82,7 @@ func TestActiveForWorkspaceReconcilesAbandonedWorkers(t *testing.T) { t.Fatalf("ReleaseWorkerLockForTest: %v", err) } - active, err := store.ActiveForWorkspace("ws-1", "up") + active, err := store.ActiveForWorkspace(workspaceOne, "up") if err != nil { t.Fatalf("ActiveForWorkspace: %v", err) } @@ -91,10 +101,10 @@ func TestActiveForWorkspaceReconcilesAbandonedWorkers(t *testing.T) { func TestActiveForWorkspaceTreatsLiveWorkerAsActive(t *testing.T) { store := newTestStore(t) - tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: "ws-1"}) + tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) holdWorkerLock(t, tk) - active, err := store.ActiveForWorkspace("ws-1", "up") + active, err := store.ActiveForWorkspace(workspaceOne, "up") if err != nil { t.Fatalf("ActiveForWorkspace: %v", err) } diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index cfa9288008..aeb418cb2c 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/devsy-org/devsy/pkg/clierr" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/status" ) @@ -363,7 +364,7 @@ func TestCancelWithoutPIDMarksFailed(t *testing.T) { if state.Error != ErrCanceled.Error() { t.Errorf("error = %q, want %q", state.Error, ErrCanceled.Error()) } - if state.ErrorCode != "canceled" || state.ErrorHint == "" { + if state.ErrorCode != string(clierr.CodeCanceled) || state.ErrorHint == "" { t.Errorf( "cancellation metadata = (%q, %q), want stable code and hint", state.ErrorCode, @@ -809,7 +810,7 @@ func TestReconcilePreservesCancellationReason(t *testing.T) { if reconciled.Error != ErrCanceled.Error() { t.Errorf("error = %q, want %q", reconciled.Error, ErrCanceled.Error()) } - if reconciled.ErrorCode != "canceled" || reconciled.ErrorHint == "" { + if reconciled.ErrorCode != string(clierr.CodeCanceled) || reconciled.ErrorHint == "" { t.Errorf( "cancellation metadata = (%q, %q), want stable code and hint", reconciled.ErrorCode, @@ -818,8 +819,8 @@ func TestReconcilePreservesCancellationReason(t *testing.T) { } } -func TestCancelKillFailureRemainsRetryable(t *testing.T) { - store := newTestStore(t) +func newLiveWorkerTask(t *testing.T, store *Store) *Task { + t.Helper() tk, err := store.Create(CreateOptions{}) if err != nil { t.Fatalf("Create: %v", err) @@ -828,6 +829,25 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { if err := tk.SetPID(4242); err != nil { t.Fatalf("SetPID: %v", err) } + return tk +} + +func requireCanceledState(t *testing.T, store *Store, id string) { + t.Helper() + state, err := store.Get(id) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Status != StatusFailed || + state.Error != ErrCanceled.Error() || + state.ErrorCode != string(clierr.CodeCanceled) { + t.Errorf("unexpected final state: %+v", state) + } +} + +func TestCancelKillFailureRemainsRetryable(t *testing.T) { + store := newTestStore(t) + tk := newLiveWorkerTask(t, store) killCalls := 0 store.SetKillProcessForTest(func(string) error { @@ -846,7 +866,10 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { t.Fatalf("Get: %v", err) } if state.Status.Terminal() { - t.Fatalf("task recorded terminal %q while its worker survived the failed kill", state.Status) + t.Fatalf( + "task recorded terminal %q while its worker survived the failed kill", + state.Status, + ) } if err := tk.Cancel(); err != nil { @@ -855,25 +878,12 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { if killCalls != 2 { t.Errorf("kill invoked %d times, want 2 (one per Cancel)", killCalls) } - state, err = store.Get(tk.ID()) - if err != nil { - t.Fatalf("Get: %v", err) - } - if state.Status != StatusFailed || state.Error != ErrCanceled.Error() || state.ErrorCode != "canceled" { - t.Errorf("unexpected final state: %+v", state) - } + requireCanceledState(t, store, tk.ID()) } func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { store := newTestStore(t) - tk, err := store.Create(CreateOptions{}) - if err != nil { - t.Fatalf("Create: %v", err) - } - holdWorkerLock(t, tk) - if err := tk.SetPID(4242); err != nil { - t.Fatalf("SetPID: %v", err) - } + tk := newLiveWorkerTask(t, store) store.SetKillProcessForTest(func(string) error { state, err := store.Get(tk.ID()) @@ -889,13 +899,7 @@ func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { if err := tk.Cancel(); err != nil { t.Fatalf("Cancel: %v", err) } - state, err := store.Get(tk.ID()) - if err != nil { - t.Fatalf("Get: %v", err) - } - if state.Status != StatusFailed || state.Error != ErrCanceled.Error() { - t.Errorf("unexpected final state: %+v", state) - } + requireCanceledState(t, store, tk.ID()) } func TestCancelDeadWorkerDoesNotInvokeKill(t *testing.T) { @@ -922,13 +926,7 @@ func TestCancelDeadWorkerDoesNotInvokeKill(t *testing.T) { if err := tk.Cancel(); err != nil { t.Fatalf("Cancel: %v", err) } - state, err := store.Get(tk.ID()) - if err != nil { - t.Fatalf("Get: %v", err) - } - if state.Status != StatusFailed || state.Error != ErrCanceled.Error() { - t.Errorf("unexpected final state: %+v", state) - } + requireCanceledState(t, store, tk.ID()) } func TestCancelRacingLateSuccessStillWinsAfterSuccessfulTermination(t *testing.T) { diff --git a/pkg/tunnel/local_listener_test.go b/pkg/tunnel/local_listener_test.go index 60158e6d3c..ce35dfc4e0 100644 --- a/pkg/tunnel/local_listener_test.go +++ b/pkg/tunnel/local_listener_test.go @@ -78,9 +78,9 @@ func listenerDialClosed(err error) bool { return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) } -func waitForListenerClosed(t *testing.T, addr string, timeout time.Duration) { +func waitForListenerClosed(t *testing.T, addr string) { t.Helper() - deadline := time.NewTimer(timeout) + deadline := time.NewTimer(2 * time.Second) defer deadline.Stop() ticker := time.NewTicker(10 * time.Millisecond) defer ticker.Stop() @@ -195,7 +195,7 @@ func TestLocalTunnel_CloseStopsAccepting(t *testing.T) { addr := tun.Addr() _ = tun.Close() - waitForListenerClosed(t, addr, 2*time.Second) + waitForListenerClosed(t, addr) } func TestLocalTunnel_ContextCancellation(t *testing.T) { @@ -216,7 +216,7 @@ func TestLocalTunnel_ContextCancellation(t *testing.T) { cancel() // The listener is closed asynchronously after cancellation. - waitForListenerClosed(t, addr, 2*time.Second) + waitForListenerClosed(t, addr) } func TestLocalTunnel_HealthCheckShutdown(t *testing.T) { @@ -238,7 +238,7 @@ func TestLocalTunnel_HealthCheckShutdown(t *testing.T) { defer func() { _ = tun.Close() }() // The health check should shut down the tunnel after 3 failures. - waitForListenerClosed(t, tun.Addr(), 2*time.Second) + waitForListenerClosed(t, tun.Addr()) } func TestLocalTunnel_HealthCheckUsesHealthFuncNotDialFunc(t *testing.T) { @@ -248,7 +248,7 @@ func TestLocalTunnel_HealthCheckUsesHealthFuncNotDialFunc(t *testing.T) { // data path, failures would never accumulate and the tunnel would stay // alive. Only a health loop driven by the failing HealthCheckFunc shuts // the listener down. - var healthCalls int32 + var healthCalls atomic.Int32 tun, err := NewLocalTunnel(ctx, LocalTunnelOptions{ BasePort: 18600, DialFunc: func(ctx context.Context) (io.ReadWriteCloser, error) { @@ -257,7 +257,7 @@ func TestLocalTunnel_HealthCheckUsesHealthFuncNotDialFunc(t *testing.T) { return remote, nil }, HealthCheckFunc: func(context.Context) error { - atomic.AddInt32(&healthCalls, 1) + healthCalls.Add(1) return fmt.Errorf("workspace gone") }, HealthCheckInterval: 50 * time.Millisecond, @@ -267,8 +267,8 @@ func TestLocalTunnel_HealthCheckUsesHealthFuncNotDialFunc(t *testing.T) { } defer func() { _ = tun.Close() }() - waitForListenerClosed(t, tun.Addr(), 2*time.Second) - if got := atomic.LoadInt32(&healthCalls); got < 3 { + waitForListenerClosed(t, tun.Addr()) + if got := healthCalls.Load(); got < 3 { t.Errorf("health probe invoked %d times, want at least 3", got) } } @@ -277,11 +277,11 @@ func TestLocalTunnel_AcceptedConnectionStillUsesDialFunc(t *testing.T) { ctx := t.Context() echoServer := newEchoServer(t) - var dialCalls int32 + var dialCalls atomic.Int32 tun, err := NewLocalTunnel(ctx, LocalTunnelOptions{ BasePort: 18700, DialFunc: func(ctx context.Context) (io.ReadWriteCloser, error) { - atomic.AddInt32(&dialCalls, 1) + dialCalls.Add(1) return net.Dial("tcp", echoServer.Addr().String()) }, HealthCheckFunc: func(context.Context) error { @@ -298,7 +298,7 @@ func TestLocalTunnel_AcceptedConnectionStillUsesDialFunc(t *testing.T) { if buf := sendAndReceive(t, tun.Addr(), msg); string(buf) != string(msg) { t.Errorf("expected %q, got %q", msg, buf) } - if got := atomic.LoadInt32(&dialCalls); got == 0 { + if got := dialCalls.Load(); got == 0 { t.Error("accepted connection did not invoke DialFunc") } } From 499e694b9c0e0bdda282b91bb5f271eaee6a06ae Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 00:17:39 -0700 Subject: [PATCH 09/49] fix(task,workspace,command): close review findings on cancel and tree teardown - Cancel awaits PID publication from a starting worker instead of recording canceled with nothing signaled; a worker that never publishes stays retryable - Detached up labels carry the normalized workspace ID so stop/delete quiescence matches tasks submitted by path - Forced delete still takes the workspace lock; only the status check is skipped, keeping the quiescence rescan serialized - Windows Job Objects are named after the task-unique worker, not the PID, so a reused PID never inherits a predecessor tree - Background processes spawn in their own process group on unix and kill signals the group, so detached descendants die with the worker - Sweep comments across the diff: names, types, and boundaries carry what they already say --- cmd/workspace/stop.go | 12 ++--- cmd/workspace/stop_test.go | 4 +- cmd/workspace/up/detach.go | 7 ++- cmd/workspace/up/detach_test.go | 15 ++++++ cmd/workspace/up/tunnel.go | 5 +- cmd/workspace/up/tunnel_test.go | 5 +- e2e/tests/ssh/detached_task.go | 6 --- e2e/tests/ssh/task_cancel.go | 3 +- pkg/command/background.go | 13 +++--- pkg/command/job_windows.go | 38 +++++++-------- pkg/command/process.go | 8 +++- pkg/command/process_supported.go | 27 ++++++++--- pkg/command/process_windows.go | 29 ++++++------ pkg/command/process_windows_test.go | 17 +++---- pkg/task/store.go | 57 +++++++++++++++++++---- pkg/task/task.go | 22 ++++++--- pkg/task/task_test.go | 66 +++++++++++++++++++++++++-- pkg/tunnel/local_listener.go | 11 ++--- pkg/tunnel/local_listener_test.go | 6 +-- pkg/workspace/delete.go | 21 +++++---- pkg/workspace/delete_progress_test.go | 19 +++++--- pkg/workspace/delete_test.go | 6 ++- pkg/workspace/quiesce.go | 8 ++-- pkg/workspace/quiesce_test.go | 2 +- 24 files changed, 272 insertions(+), 135 deletions(-) diff --git a/cmd/workspace/stop.go b/cmd/workspace/stop.go index 1d70afddaa..b4668e83e1 100644 --- a/cmd/workspace/stop.go +++ b/cmd/workspace/stop.go @@ -22,9 +22,8 @@ import ( type StopCmd struct { *flags.GlobalFlags client2.StopOptions - // quiesceUpTasks cancels the workspace's persisted detached up tasks; - // nil resolves the default task store. Tests inject it to drive - // cancellation outcomes without real worker processes. + // Test seam; see task.Store.SetKillProcessForTest. nil resolves the + // default store. quiesceUpTasks func(workspaceID string) error } @@ -147,10 +146,9 @@ func (cmd *StopCmd) run( defer client.Unlock() } - // Quiesce again under the lock: task submission and lock acquisition are - // not atomic, so a task may have become visible while the lock wait - // blocked. A surviving up worker can restart the workspace after the - // stop, so failing here must fail the stop. + // Quiesce again under the lock: a task may have become visible while + // the lock wait blocked, and a surviving up worker can restart the + // workspace after the stop. if err := cmd.quiesce(client.Workspace()); err != nil { return err } diff --git a/cmd/workspace/stop_test.go b/cmd/workspace/stop_test.go index 903c7af123..4afd76ae30 100644 --- a/cmd/workspace/stop_test.go +++ b/cmd/workspace/stop_test.go @@ -96,8 +96,8 @@ func TestStopFindsTaskWithoutDesktopState(t *testing.T) { config.ResetPathManager() t.Cleanup(config.ResetPathManager) - // A persisted detached up task with no live worker, as left behind after - // a Desktop restart: nothing in memory references it. + // A persisted up task with no live worker, as a Desktop restart leaves + // behind. store, err := task.NewStore() require.NoError(t, err) tk, err := store.Create(task.CreateOptions{Command: "up", WorkspaceID: testWorkspaceName}) diff --git a/cmd/workspace/up/detach.go b/cmd/workspace/up/detach.go index 4f03ef5506..34cc1ee6b4 100644 --- a/cmd/workspace/up/detach.go +++ b/cmd/workspace/up/detach.go @@ -11,6 +11,7 @@ import ( "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/output" "github.com/devsy-org/devsy/pkg/task" + workspace2 "github.com/devsy-org/devsy/pkg/workspace" ) // runDetached submits this invocation as a background task and returns @@ -76,13 +77,15 @@ func detachedArgs(args []string) []string { return out } -// detachWorkspaceLabel is a best-effort label for task list. +// detachWorkspaceLabel resolves the workspace ID at submission so stop and +// delete can match the task before the worker starts and records the +// authoritative ID. func (cmd *UpCmd) detachWorkspaceLabel(args []string) string { if cmd.ID != "" { return cmd.ID } if len(args) > 0 { - return args[0] + return workspace2.ToID(args[0]) } return "" } diff --git a/cmd/workspace/up/detach_test.go b/cmd/workspace/up/detach_test.go index 965d7b5551..4ed73804e6 100644 --- a/cmd/workspace/up/detach_test.go +++ b/cmd/workspace/up/detach_test.go @@ -3,6 +3,8 @@ package up import ( "reflect" "testing" + + workspace2 "github.com/devsy-org/devsy/pkg/workspace" ) const ( @@ -33,3 +35,16 @@ func TestDetachedArgs_NoDetachFlagIsUnchanged(t *testing.T) { t.Errorf("detachedArgs() = %v, want %v", got, want) } } + +func TestDetachWorkspaceLabelNormalizesPath(t *testing.T) { + cmd := &UpCmd{} + raw := "/tmp/some workspace dir" + got := cmd.detachWorkspaceLabel([]string{raw}) + want := workspace2.ToID(raw) + if got != want { + t.Errorf("detachWorkspaceLabel() = %q, want %q", got, want) + } + if got == raw { + t.Error("label must be the workspace ID, not the raw path") + } +} diff --git a/cmd/workspace/up/tunnel.go b/cmd/workspace/up/tunnel.go index 82ce300261..213e78e24a 100644 --- a/cmd/workspace/up/tunnel.go +++ b/cmd/workspace/up/tunnel.go @@ -11,9 +11,8 @@ import ( ) // workspaceTunnelHealth probes whether the workspace behind the tunnel is -// still running. It is strictly observational: it must never dial the SSH -// path or otherwise start the workspace, because a background health check -// that can revive a stopped workspace reverses deliberate stops. +// still running. It must stay observational: a health check that dials the +// SSH path can revive a stopped workspace, reversing deliberate stops. func workspaceTunnelHealth( client client2.BaseWorkspaceClient, ) func(ctx context.Context) error { diff --git a/cmd/workspace/up/tunnel_test.go b/cmd/workspace/up/tunnel_test.go index 42ad6d3b65..2ec2b2ce65 100644 --- a/cmd/workspace/up/tunnel_test.go +++ b/cmd/workspace/up/tunnel_test.go @@ -8,9 +8,8 @@ import ( client2 "github.com/devsy-org/devsy/pkg/client" ) -// statusOnlyClient answers Status and panics on any other method through the -// embedded nil interface, so a health probe that reaches for a mutating path -// fails the test. +// statusOnlyClient panics on any method but Status, so a health probe that +// reaches for a mutating path fails the test. type statusOnlyClient struct { client2.BaseWorkspaceClient status client2.Status diff --git a/e2e/tests/ssh/detached_task.go b/e2e/tests/ssh/detached_task.go index 58807c4b8f..2456a6d92b 100644 --- a/e2e/tests/ssh/detached_task.go +++ b/e2e/tests/ssh/detached_task.go @@ -39,8 +39,6 @@ func setupTunnelProvider(ctx context.Context, initialDir string) *framework.Fram return f } -// startDetachedTunnelUp submits `workspace up --detach --ssh-tunnel` and -// returns the task ID from the envelope on stdout. func startDetachedTunnelUp( ctx context.Context, f *framework.Framework, @@ -97,8 +95,6 @@ func findDetachedUpTask( return detachedUpState{}, fmt.Errorf("task %s not in task list", taskID) } -// waitDetachedTunnelReady polls the task until the up pipeline reports the -// ready phase, which it reaches only with the tunnel active. func waitDetachedTunnelReady( ctx context.Context, f *framework.Framework, @@ -120,8 +116,6 @@ func waitDetachedTunnelReady( return state } -// waitDetachedTaskCanceled polls until the task records the canceled error -// code and returns the final state. func waitDetachedTaskCanceled( ctx context.Context, f *framework.Framework, diff --git a/e2e/tests/ssh/task_cancel.go b/e2e/tests/ssh/task_cancel.go index 2c3abfeb4b..0ff22875c6 100644 --- a/e2e/tests/ssh/task_cancel.go +++ b/e2e/tests/ssh/task_cancel.go @@ -54,8 +54,7 @@ var _ = ginkgo.Describe( gomega.Expect(active.PID).NotTo(gomega.BeZero(), "detached up worker should publish its pid") - // Cancellation goes through the product task API; on Windows - // this used to panic with "unsupported". + // On Windows this used to panic with "unsupported". framework.ExpectNoError(f.DevsyWorkspaceTaskCancel(ctx, taskID)) canceled := waitDetachedTaskCanceled(ctx, f, taskID) diff --git a/pkg/command/background.go b/pkg/command/background.go index ee1f046302..3b39d62670 100644 --- a/pkg/command/background.go +++ b/pkg/command/background.go @@ -59,7 +59,7 @@ func StartBackgroundOnce(commandName string, createCommand CreateCommand) error return err } - return startCommand(cmd, paths.pidFile, paths.streamsFile) + return startCommand(cmd, commandName, paths.pidFile, paths.streamsFile) } type processPaths struct { @@ -98,26 +98,27 @@ func StartBackground(commandName string, createCommand CreateCommand) error { return err } - return startDetached(cmd, "", streamsFile) + return startDetached(cmd, commandName, "", streamsFile) } -func startCommand(cmd *exec.Cmd, pidFile, streamsFile string) error { - return startDetached(cmd, pidFile, streamsFile) +func startCommand(cmd *exec.Cmd, commandName, pidFile, streamsFile string) error { + return startDetached(cmd, commandName, pidFile, streamsFile) } -func startDetached(cmd *exec.Cmd, pidFile, streamsFile string) error { +func startDetached(cmd *exec.Cmd, commandName, pidFile, streamsFile string) error { streamsF, err := openStreamsFile(cmd, streamsFile) if err != nil { return err } + prepareBackgroundTree(cmd) if err := cmd.Start(); err != nil { closeFile(streamsF) return fmt.Errorf("start process: %w", err) } closeFile(streamsF) - if err := ownProcessTree(cmd.Process.Pid); err != nil { + if err := ownProcessTree(cmd.Process.Pid, commandName); err != nil { fmt.Fprintf( os.Stderr, "warning: process tree ownership unavailable for pid %d: %v\n", diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index e8f4e64d63..a43e0495e8 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -5,7 +5,6 @@ package command import ( "errors" "fmt" - "strconv" "unsafe" "golang.org/x/sys/windows" @@ -15,21 +14,18 @@ const jobObjectTerminateAccess = 0x0008 var procOpenJobObjectW = windows.NewLazySystemDLL("kernel32.dll").NewProc("OpenJobObjectW") -// jobNameForPID names the Job Object owning the process tree rooted at pid, -// so cancellation can recover the job from the persisted PID alone. -func jobNameForPID(pid int) string { - return "devsy-worker-" + strconv.Itoa(pid) +// jobNameFor names the Job Object after the worker (e.g. devsy-up-) +// rather than its PID: a reused PID must never inherit a predecessor's job. +func jobNameFor(name string) string { + return name } -// ownProcessTree assigns the process to a Job Object named after its PID. -// Parent-based termination (taskkill /T) loses track of descendants whose -// intermediate parent already exited; job membership survives that, so tree -// teardown stays complete. The launcher closes its handle right away: the -// job lives as long as it has member processes, which is what keeps the -// detached worker alive after the launcher exits. Assignment failure is -// non-fatal; termination then falls back to taskkill /T /F. -func ownProcessTree(pid int) error { - name, err := windows.UTF16PtrFromString(jobNameForPID(pid)) +// ownProcessTree assigns the process to a Job Object named for the worker. +// Job membership outlives intermediate parents, so tree teardown stays +// complete where taskkill /T loses track. Assignment failure is non-fatal; +// termination then falls back to taskkill /T /F. +func ownProcessTree(pid int, workerName string) error { + name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) if err != nil { return err } @@ -55,11 +51,11 @@ func ownProcessTree(pid int) error { return nil } -// terminateJob kills the process tree owned by the Job Object named for pid. -// The bool reports whether a job existed; without one the caller falls back -// to taskkill, e.g. for workers launched before job ownership existed. -func terminateJob(pid int) (bool, error) { - name, err := windows.UTF16PtrFromString(jobNameForPID(pid)) +// terminateJob kills the tree owned by the worker's Job Object. The bool +// reports whether a job existed; without one the caller falls back to +// taskkill, e.g. for workers launched before job ownership existed. +func terminateJob(workerName string) (bool, error) { + name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) if err != nil { return false, err } @@ -72,13 +68,13 @@ func terminateJob(pid int) (bool, error) { if errors.Is(callErr, windows.ERROR_FILE_NOT_FOUND) { return false, nil } - return false, fmt.Errorf("open job for pid %d: %w", pid, callErr) + return false, fmt.Errorf("open job %s: %w", workerName, callErr) } job := windows.Handle(handle) defer func() { _ = windows.CloseHandle(job) }() if err := windows.TerminateJobObject(job, 1); err != nil { - return false, fmt.Errorf("terminate job for pid %d: %w", pid, err) + return false, fmt.Errorf("terminate job %s: %w", workerName, err) } return true, nil } diff --git a/pkg/command/process.go b/pkg/command/process.go index 4eae843f56..fb1d422594 100644 --- a/pkg/command/process.go +++ b/pkg/command/process.go @@ -5,5 +5,11 @@ func IsRunning(pid string) (bool, error) { } func Kill(pid string) error { - return kill(pid) + return killTree(pid, "") +} + +// KillTree terminates the process and everything it spawned: the job or +// process group treeName identifies when set, otherwise the process alone. +func KillTree(pid, treeName string) error { + return killTree(pid, treeName) } diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index eba3bce5bd..955adfd4d6 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -5,6 +5,7 @@ package command import ( "errors" "os" + "os/exec" "strconv" "syscall" "time" @@ -29,28 +30,42 @@ func isRunning(pid string) (bool, error) { return true, nil } -func kill(pid string) error { +// A background process that leads its own group (prepareBackgroundTree) is +// signaled group-wide so descendants that outlive it still die with it. +func killTree(pid, _ string) error { parsedPid, err := strconv.Atoi(pid) if err != nil { return err } - if err := syscall.Kill(parsedPid, syscall.SIGTERM); err != nil { + target := parsedPid + if pgid, err := syscall.Getpgid(parsedPid); err == nil && pgid == parsedPid { + target = -parsedPid + } + + if err := syscall.Kill(target, syscall.SIGTERM); err != nil { if errors.Is(err, syscall.ESRCH) { return nil // already exited } return err } time.Sleep(2 * time.Second) - err = syscall.Kill(parsedPid, syscall.SIGKILL) + err = syscall.Kill(target, syscall.SIGKILL) if err != nil && !errors.Is(err, syscall.ESRCH) { return err } return nil } -// ownProcessTree is a no-op on Unix: process-group termination via the -// worker's PID already covers the tree, and Job Objects are Windows-only. -func ownProcessTree(pid int) error { +// ownProcessTree is a no-op on Unix: the process group created by +// prepareBackgroundTree already scopes the tree. +func ownProcessTree(pid int, name string) error { return nil } + +func prepareBackgroundTree(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setpgid = true +} diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index 74e36bc022..0494f3de43 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -43,7 +43,7 @@ func isRunning(pid string) (bool, error) { return exitCode == stillActive, nil } -func kill(pid string) error { +func killTree(pid, treeName string) error { parsed, err := parsePID(pid) if err != nil { return err @@ -57,17 +57,16 @@ func kill(pid string) error { return nil } - // Job Object termination tears down the whole tree, including orphans - // that parent-based termination cannot reach. Workers launched before - // job ownership existed fall through to taskkill. - if terminated, jobErr := terminateJob(parsed); jobErr == nil && terminated { - return verifyTerminated(parsed) + // Job Object termination reaches orphaned descendants; workers launched + // before job ownership existed fall through to taskkill. + if treeName != "" { + if terminated, jobErr := terminateJob(treeName); jobErr == nil && terminated { + return verifyTerminated(parsed) + } } - // /T takes down the worker's descendants as well: a detached up worker - // can spawn children (e.g. devsy workspace ssh --stdio) that keep owning - // transport resources. /F stands in for SIGKILL; Windows has no SIGTERM - // equivalent for arbitrary processes. + // /T takes down the worker's descendants; /F stands in for SIGKILL, + // which Windows lacks for arbitrary processes. cmd := exec.Command("taskkill", "/PID", strconv.Itoa(parsed), "/T", "/F") output, runErr := cmd.CombinedOutput() if runErr != nil { @@ -87,8 +86,10 @@ func kill(pid string) error { return verifyTerminated(parsed) } -// verifyTerminated confirms the process is gone, allowing a brief window for -// the exit to become visible after the termination call reported success. +func prepareBackgroundTree(*exec.Cmd) {} + +// verifyTerminated allows a brief window for the exit to become visible +// after the termination call reported success. func verifyTerminated(parsed int) error { pid := strconv.Itoa(parsed) deadline := time.Now().Add(5 * time.Second) @@ -118,8 +119,8 @@ func parsePID(pid string) (int, error) { return parsed, nil } -// truncateOutput bounds taskkill's captured output; the text itself is -// localized and never parsed. +// truncateOutput bounds captured taskkill output, which is localized and +// never parsed. func truncateOutput(output string) string { const maxOutput = 256 if len(output) > maxOutput { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 9659709c4c..ce0680f830 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -21,8 +21,7 @@ const ( ) // TestHelperProcess is not a test; every helper process re-executes the test -// binary into it. The sleep loop keeps Go's deadlock detector quiet while the -// helper waits to be killed. +// binary into it. func TestHelperProcess(t *testing.T) { if os.Getenv(helperEnvMarker) != "1" { return @@ -63,9 +62,8 @@ func TestHelperProcess(t *testing.T) { } } -// helperEnv strips helper markers from the inherited environment so only the -// process being launched gets them. Windows process creation needs the rest -// of the environment (e.g. SYSTEMROOT) intact. +// helperEnv strips helper markers from the inherited environment; Windows +// process creation needs the rest (e.g. SYSTEMROOT) intact. func helperEnv(extra ...string) []string { env := []string{} for _, kv := range os.Environ() { @@ -191,7 +189,7 @@ func TestInvalidPIDReturnsError(t *testing.T) { if _, err := isRunning(pid); err == nil { t.Errorf("isRunning(%q) = nil error, want rejection", pid) } - if err := kill(pid); err == nil { + if err := killTree(pid, ""); err == nil { t.Errorf("kill(%q) = nil, want error", pid) } } @@ -203,15 +201,14 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { childPIDFile := filepath.Join(dir, "child.pid") grandchildPIDFile := filepath.Join(dir, "grandchild.pid") - // The parent waits so the job can own it before it spawns anything; the - // child then spawns the grandchild and exits, orphaning the grandchild + // The child spawns the grandchild and exits, orphaning the grandchild // from the perspective of parent-based termination. parent := startHelper(t, helperEnvWaitFile+"="+waitFile, helperEnvPIDFile+"="+childPIDFile, helperEnvChildPIDFile+"="+grandchildPIDFile, ) - if err := ownProcessTree(parent.Process.Pid); err != nil { + if err := ownProcessTree(parent.Process.Pid, "devsy-test-worker"); err != nil { t.Skipf("job assignment unavailable in this environment: %v", err) } if err := os.WriteFile(waitFile, []byte("go"), 0o600); err != nil { @@ -224,7 +221,7 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { assertRunning(t, parent.Process.Pid) assertRunning(t, grandchildPID) - if err := Kill(strconv.Itoa(parent.Process.Pid)); err != nil { + if err := KillTree(strconv.Itoa(parent.Process.Pid), "devsy-test-worker"); err != nil { t.Fatalf("Kill(parent): %v", err) } diff --git a/pkg/task/store.go b/pkg/task/store.go index 6dc0348885..87afef0c37 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -18,11 +18,15 @@ import ( // lockTimeout bounds how long update waits to acquire a task's file lock. const lockTimeout = 5 * time.Second +// pidPublishTimeout bounds Cancel's wait for a starting worker to publish +// its PID. +const pidPublishTimeout = 5 * time.Second + // Store persists task state as one JSON file per task under dir. type Store struct { dir string // Test seam; see Store.SetKillProcessForTest. - killProcess func(string) error + killProcess func(pid, treeName string) error // Test seam; see Store.SetAfterClaimForTest. afterClaimForTest func() } @@ -39,7 +43,7 @@ func NewStoreAt(dir string) (*Store, error) { if err := os.MkdirAll(dir, 0o750); err != nil { return nil, fmt.Errorf("create task dir: %w", err) } - return &Store{dir: dir, killProcess: command.Kill}, nil + return &Store{dir: dir, killProcess: command.KillTree}, nil } func (s *Store) Create(opts CreateOptions) (*Task, error) { @@ -173,8 +177,7 @@ func (s *Store) List() ([]*State, error) { // ActiveForWorkspace returns the non-terminal tasks labeled with the exact // workspace ID, newest first, reconciling stale workers first so an // abandoned task does not read as live. A non-empty command restricts the -// match to that command. The store only answers which persisted operations -// are still active; callers decide what to do with them. +// match. func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error) { states, err := s.List() if err != nil { @@ -198,13 +201,28 @@ func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error return active, nil } +// awaitPID polls for the worker to publish its PID, which it does right +// after claiming the worker lock. +func (s *Store) awaitPID(id string, timeout time.Duration) int { + deadline := time.Now().Add(timeout) + for { + state, err := s.Get(id) + if err == nil && state.PID != 0 { + return state.PID + } + if time.Now().After(deadline) { + return 0 + } + time.Sleep(50 * time.Millisecond) + } +} + // SetKillProcessForTest replaces this store's process termination hook, so a // test can drive cancellation outcomes deterministically without real PIDs. -// Production code must never call it. // -// Not in export_test.go: other packages' tests need it, and a _test.go file is -// only compiled into its own package's test binary. -func (s *Store) SetKillProcessForTest(fn func(string) error) { +// Not in export_test.go: other packages' tests need it, and a _test.go file +// compiles only into its own package's test binary. +func (s *Store) SetKillProcessForTest(fn func(pid, treeName string) error) { s.killProcess = fn } @@ -239,6 +257,29 @@ func (s *Store) workerAlive(id string) bool { return false } +// workerStarting reports whether a live worker holds the task's worker lock +// without having published its PID yet: Cancel then has nothing to signal +// until the worker publishes. +func (s *Store) workerStarting(id string) bool { + path, err := s.workerLockPath(id) + if err != nil { + return false + } + if _, err := os.Stat(path); err != nil { + return false + } + lock := flock.New(path) + locked, err := lock.TryLock() + if err != nil { + return false + } + if locked { + _ = lock.Unlock() + return false + } + return true +} + // claimDeadWorkerLock acquires the task's worker lock, which only succeeds when // no worker holds it. func (s *Store) claimDeadWorkerLock(id string) (*flock.Flock, bool) { diff --git a/pkg/task/task.go b/pkg/task/task.go index 026ede3721..8aaf2da5e0 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -160,11 +160,11 @@ func (t *Task) Succeed(result *config.Result) error { } // Cancel terminates the task's worker before recording the task as canceled: -// the persisted terminal state must never claim the operation is gone while -// the worker that performs it is still alive. A kill failure leaves the task -// nonterminal so a later Cancel retries. Cancellation is idempotent: a task -// already terminal, or whose worker already exited, only (re)finalizes the -// canceled state. +// the persisted state must never claim the operation is gone while the worker +// performing it is still alive. A kill failure leaves the task nonterminal so +// a later Cancel retries. Cancellation is idempotent: a task already +// terminal, or whose worker already exited, only (re)finalizes the canceled +// state. func (t *Task) Cancel() error { var pid int observedNonterminal := false @@ -182,8 +182,18 @@ func (t *Task) Cancel() error { return nil } + if pid == 0 && t.store.workerStarting(t.id) { + pid = t.store.awaitPID(t.id, pidPublishTimeout) + if pid == 0 { + return fmt.Errorf( + "cancel task %s: worker has not published its pid yet, retry", + t.id, + ) + } + } + if pid != 0 && t.store.workerAlive(t.id) { - if err := t.store.killProcess(strconv.Itoa(pid)); err != nil { + if err := t.store.killProcess(strconv.Itoa(pid), WorkerProcessName(t.id)); err != nil { if !t.store.workerAlive(t.id) { // The worker exited during the kill attempt, so termination // is satisfied even though the kill itself reported an error. diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index aeb418cb2c..aa16f3e404 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -850,7 +850,7 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { tk := newLiveWorkerTask(t, store) killCalls := 0 - store.SetKillProcessForTest(func(string) error { + store.SetKillProcessForTest(func(pid, treeName string) error { killCalls++ if killCalls == 1 { return errors.New("boom") @@ -885,7 +885,7 @@ func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { store := newTestStore(t) tk := newLiveWorkerTask(t, store) - store.SetKillProcessForTest(func(string) error { + store.SetKillProcessForTest(func(pid, treeName string) error { state, err := store.Get(tk.ID()) if err != nil { t.Errorf("Get during kill: %v", err) @@ -902,6 +902,64 @@ func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { requireCanceledState(t, store, tk.ID()) } +func TestCancelAwaitsPIDPublication(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + holdWorkerLock(t, tk) + + var killedPID, killedTree string + store.SetKillProcessForTest(func(pid, treeName string) error { + killedPID, killedTree = pid, treeName + return nil + }) + + go func() { + time.Sleep(200 * time.Millisecond) + if err := tk.SetPID(4242); err != nil { + t.Errorf("SetPID: %v", err) + } + }() + + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + if killedPID != "4242" { + t.Errorf("kill pid = %q, want 4242", killedPID) + } + if killedTree != WorkerProcessName(tk.ID()) { + t.Errorf("kill tree = %q, want %q", killedTree, WorkerProcessName(tk.ID())) + } + requireCanceledState(t, store, tk.ID()) +} + +func TestCancelUnpublishedPIDStaysRetryable(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + holdWorkerLock(t, tk) + + store.SetKillProcessForTest(func(pid, treeName string) error { + t.Error("kill invoked before the worker published a pid") + return nil + }) + + if err := tk.Cancel(); err == nil { + t.Fatal("Cancel = nil, want unpublished-pid error") + } + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Status.Terminal() { + t.Fatalf("task recorded terminal %q with no killed worker", state.Status) + } +} + func TestCancelDeadWorkerDoesNotInvokeKill(t *testing.T) { store := newTestStore(t) tk, err := store.Create(CreateOptions{}) @@ -918,7 +976,7 @@ func TestCancelDeadWorkerDoesNotInvokeKill(t *testing.T) { t.Fatalf("SetPID: %v", err) } - store.SetKillProcessForTest(func(string) error { + store.SetKillProcessForTest(func(pid, treeName string) error { t.Error("kill invoked for a task whose worker lock is dead") return nil }) @@ -940,7 +998,7 @@ func TestCancelRacingLateSuccessStillWinsAfterSuccessfulTermination(t *testing.T t.Fatalf("SetPID: %v", err) } - store.SetKillProcessForTest(func(string) error { + store.SetKillProcessForTest(func(pid, treeName string) error { // The worker reports its own success while the kill unwinds it. return tk.Succeed(&config.Result{}) }) diff --git a/pkg/tunnel/local_listener.go b/pkg/tunnel/local_listener.go index c59bd68e7b..9f9707c1e3 100644 --- a/pkg/tunnel/local_listener.go +++ b/pkg/tunnel/local_listener.go @@ -32,13 +32,12 @@ type DialFunc func(ctx context.Context) (io.ReadWriteCloser, error) type LocalTunnelOptions struct { // BasePort is the starting port to search from (default: 10800) BasePort int - // DialFunc creates a connection to the remote SSH endpoint. It is the - // data path and runs only for real accepted local connections. + // DialFunc is the data path: it connects to the remote SSH endpoint + // for accepted local connections. DialFunc DialFunc - // HealthCheckFunc is the control-plane liveness probe. It must be - // observational: it must never start, restart, or otherwise mutate the - // resource it probes. When nil, no periodic probe runs and the listener - // lifetime follows its context. + // HealthCheckFunc is the control-plane liveness probe. It must stay + // observational: never start, restart, or otherwise mutate the resource + // it probes. When nil, the listener lifetime follows its context. HealthCheckFunc func(ctx context.Context) error // HealthCheckInterval overrides the default health check interval (for testing). // If zero, defaults to healthCheckInterval (30s). diff --git a/pkg/tunnel/local_listener_test.go b/pkg/tunnel/local_listener_test.go index ce35dfc4e0..c022436fc2 100644 --- a/pkg/tunnel/local_listener_test.go +++ b/pkg/tunnel/local_listener_test.go @@ -244,10 +244,8 @@ func TestLocalTunnel_HealthCheckShutdown(t *testing.T) { func TestLocalTunnel_HealthCheckUsesHealthFuncNotDialFunc(t *testing.T) { ctx := t.Context() - // DialFunc always succeeds: if the health loop still probed through the - // data path, failures would never accumulate and the tunnel would stay - // alive. Only a health loop driven by the failing HealthCheckFunc shuts - // the listener down. + // DialFunc always succeeds: only a health loop driven by the failing + // HealthCheckFunc may shut the listener down. var healthCalls atomic.Int32 tun, err := NewLocalTunnel(ctx, LocalTunnelOptions{ BasePort: 18600, diff --git a/pkg/workspace/delete.go b/pkg/workspace/delete.go index 219e73da64..d3ede1748f 100644 --- a/pkg/workspace/delete.go +++ b/pkg/workspace/delete.go @@ -26,9 +26,8 @@ type DeleteOptions struct { Force bool ClientDelete client2.DeleteOptions Owner platform.OwnerFilter - // quiesceUpTasks cancels the workspace's persisted detached up tasks; - // nil resolves the default task store. Tests inject it to drive - // cancellation outcomes without real worker processes. + // Test seam; see task.Store.SetKillProcessForTest. nil resolves the + // default store. quiesceUpTasks func(workspaceID string) error } @@ -148,8 +147,10 @@ func stopIfRunning( } } -// checkBeforeDelete acquires the lock and verifies the workspace exists -// unless force-deletion is requested. It returns an unlock function that +// checkBeforeDelete acquires the lock and verifies the workspace exists; +// force-deletion skips only the status check, never the lock, so the +// quiescence rescan stays serialized with the delete. It returns an unlock +// function that // must be called by the caller (typically deferred) to release the lock, // and the resolved workspace status so the caller can decide whether a // stop is required before delete. @@ -158,16 +159,16 @@ func checkBeforeDelete( client client2.BaseWorkspaceClient, opts DeleteOptions, ) (func(), client2.Status, error) { - force := opts.Force || opts.ClientDelete.Force - if force { - return func() {}, "", nil - } - unlock, err := lockIfNeeded(ctx, client, opts) if err != nil { return nil, "", err } + force := opts.Force || opts.ClientDelete.Force + if force { + return unlock, "", nil + } + var status client2.Status err = progress.RunStep( ctx, diff --git a/pkg/workspace/delete_progress_test.go b/pkg/workspace/delete_progress_test.go index 6e512f34da..5812e232ac 100644 --- a/pkg/workspace/delete_progress_test.go +++ b/pkg/workspace/delete_progress_test.go @@ -12,10 +12,11 @@ import ( type progressDeleteClient struct { client.BaseWorkspaceClient - lock func(context.Context) error - state client.Status - stopErr error - unlocked bool + lock func(context.Context) error + state client.Status + statusCalled bool + stopErr error + unlocked bool } func (c *progressDeleteClient) Lock(ctx context.Context) error { return c.lock(ctx) } @@ -25,6 +26,7 @@ func (c *progressDeleteClient) Status( context.Context, client.StatusOptions, ) (client.Status, error) { + c.statusCalled = true return c.state, nil } func (c *progressDeleteClient) Stop(context.Context, client.StopOptions) error { return c.stopErr } @@ -83,12 +85,15 @@ func TestDeleteProgressPreservesBestEffortStop(t *testing.T) { require.Equal(t, events[0].OperationID, events[1].ParentOperationID) } -func TestDeleteForceSkipsLockAndStatus(t *testing.T) { - c := &progressDeleteClient{} +func TestDeleteForceLocksButSkipsStatus(t *testing.T) { + c := &progressDeleteClient{ + lock: func(context.Context) error { return nil }, + } unlock, _, err := checkBeforeDelete(context.Background(), c, DeleteOptions{Force: true}) require.NoError(t, err) unlock() - require.False(t, c.unlocked) + require.True(t, c.unlocked, "forced delete must still release the workspace lock") + require.False(t, c.statusCalled, "forced delete skips only the status check") } func TestDeleteLockFailureReportsFailure(t *testing.T) { diff --git a/pkg/workspace/delete_test.go b/pkg/workspace/delete_test.go index 36409f165e..79284ad620 100644 --- a/pkg/workspace/delete_test.go +++ b/pkg/workspace/delete_test.go @@ -109,8 +109,10 @@ func (c *preResolvedDeleteClient) Workspace() string { return c.workspaceID } func (c *preResolvedDeleteClient) WorkspaceConfig() *provider.Workspace { return c.config } -func (c *preResolvedDeleteClient) Context() string { return c.config.Context } -func (c *preResolvedDeleteClient) Provider() string { return DefaultDockerCommand } +func (c *preResolvedDeleteClient) Context() string { return c.config.Context } +func (c *preResolvedDeleteClient) Lock(context.Context) error { return nil } +func (c *preResolvedDeleteClient) Unlock() {} +func (c *preResolvedDeleteClient) Provider() string { return DefaultDockerCommand } func (c *preResolvedDeleteClient) Delete(context.Context, client.DeleteOptions) error { c.deleted = true return nil diff --git a/pkg/workspace/quiesce.go b/pkg/workspace/quiesce.go index f662f7036a..a89c73d337 100644 --- a/pkg/workspace/quiesce.go +++ b/pkg/workspace/quiesce.go @@ -14,8 +14,8 @@ const upTaskCommand = "up" // QuiesceUpTasks cancels every persisted, still-active detached up task for // the workspace so no surviving worker can mutate or restart it during // stop/delete. Every matching task is attempted even when one fails; -// failures are aggregated, never logged and ignored, because a surviving -// worker is a known local mutator capable of reversing the lifecycle change. +// failures are aggregated, since a surviving worker can reverse the +// lifecycle change. func QuiesceUpTasks(store *task.Store, workspaceID string) error { active, err := store.ActiveForWorkspace(workspaceID, upTaskCommand) if err != nil { @@ -39,8 +39,8 @@ func QuiesceUpTasks(store *task.Store, workspaceID string) error { return errors.Join(errs...) } -// QuiesceUpTasksForWorkspace resolves the default task store and quiesces -// the workspace's active up tasks. +// QuiesceUpTasksForWorkspace quiesces the workspace's active up tasks +// against the default task store. func QuiesceUpTasksForWorkspace(workspaceID string) error { store, err := task.NewStore() if err != nil { diff --git a/pkg/workspace/quiesce_test.go b/pkg/workspace/quiesce_test.go index 475a77a8fe..a45d838d5a 100644 --- a/pkg/workspace/quiesce_test.go +++ b/pkg/workspace/quiesce_test.go @@ -65,7 +65,7 @@ func TestQuiesceUpTasksAttemptsEveryTaskAndAggregatesFailures(t *testing.T) { } require.NoError(t, failing.SetPID(1111)) require.NoError(t, succeeding.SetPID(2222)) - store.SetKillProcessForTest(func(pid string) error { + store.SetKillProcessForTest(func(pid, treeName string) error { if pid == "1111" { return errors.New("boom") } From ecdf9dfbebd3f8e1b2ba5cd01d719e34639784df Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 00:31:29 -0700 Subject: [PATCH 10/49] fix(task,up): abort workers canceled before they claim their lock - openTask re-reads state after claiming the worker lock: a task canceled in the submit-to-lock window exits with ErrCanceled instead of running up behind a canceled state - Move awaitPID below exported methods (funcorder) --- cmd/workspace/up/detach.go | 10 ++++++++++ cmd/workspace/up/detach_test.go | 35 +++++++++++++++++++++++++++++++++ pkg/task/store.go | 18 ++++++++--------- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/cmd/workspace/up/detach.go b/cmd/workspace/up/detach.go index 34cc1ee6b4..7c8b2a492d 100644 --- a/cmd/workspace/up/detach.go +++ b/cmd/workspace/up/detach.go @@ -117,6 +117,16 @@ func (cmd *UpCmd) openTask() (*task.Task, error) { failTask(t, err) return nil, err } + state, err := store.Get(cmd.taskID) + if err != nil { + failTask(t, err) + return nil, err + } + // Canceled before this worker claimed its lock: the canceled state is + // already recorded and this worker must not overwrite it or run up. + if state.Status.Terminal() { + return nil, task.ErrCanceled + } return t, nil } diff --git a/cmd/workspace/up/detach_test.go b/cmd/workspace/up/detach_test.go index 4ed73804e6..7decfb8189 100644 --- a/cmd/workspace/up/detach_test.go +++ b/cmd/workspace/up/detach_test.go @@ -1,9 +1,12 @@ package up import ( + "errors" "reflect" "testing" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/task" workspace2 "github.com/devsy-org/devsy/pkg/workspace" ) @@ -48,3 +51,35 @@ func TestDetachWorkspaceLabelNormalizesPath(t *testing.T) { t.Error("label must be the workspace ID, not the raw path") } } + +func TestOpenTaskExitsWhenAlreadyCanceled(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + config.ResetPathManager() + t.Cleanup(config.ResetPathManager) + + store, err := task.NewStore() + if err != nil { + t.Fatalf("NewStore: %v", err) + } + tk, err := store.Create(task.CreateOptions{Command: "up"}) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + + cmd := &UpCmd{taskID: tk.ID()} + got, err := cmd.openTask() + if got != nil || !errors.Is(err, task.ErrCanceled) { + t.Fatalf("openTask() = (%v, %v), want (nil, ErrCanceled)", got, err) + } + + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Error != task.ErrCanceled.Error() { + t.Errorf("openTask overwrote the canceled state: %+v", state) + } +} diff --git a/pkg/task/store.go b/pkg/task/store.go index 87afef0c37..1f5c53010d 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -201,6 +201,15 @@ func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error return active, nil } +// SetKillProcessForTest replaces this store's process termination hook, so a +// test can drive cancellation outcomes deterministically without real PIDs. +// +// Not in export_test.go: other packages' tests need it, and a _test.go file +// compiles only into its own package's test binary. +func (s *Store) SetKillProcessForTest(fn func(pid, treeName string) error) { + s.killProcess = fn +} + // awaitPID polls for the worker to publish its PID, which it does right // after claiming the worker lock. func (s *Store) awaitPID(id string, timeout time.Duration) int { @@ -217,15 +226,6 @@ func (s *Store) awaitPID(id string, timeout time.Duration) int { } } -// SetKillProcessForTest replaces this store's process termination hook, so a -// test can drive cancellation outcomes deterministically without real PIDs. -// -// Not in export_test.go: other packages' tests need it, and a _test.go file -// compiles only into its own package's test binary. -func (s *Store) SetKillProcessForTest(fn func(pid, treeName string) error) { - s.killProcess = fn -} - // failAbandoned records ErrAbandoned for a task whose worker is confirmed gone. // Must be called with the worker lock held. func (s *Store) failAbandoned(state *State) *State { From 9df825d2c4819b6398877c3b191ddf7f2e7767dc Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 00:48:50 -0700 Subject: [PATCH 11/49] test(up): root the task store under DEVSY_HOME The Windows path manager reads %LOCALAPPDATA%, not $HOME, so the test would have touched the developer real task store on Windows. --- cmd/workspace/up/detach_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/workspace/up/detach_test.go b/cmd/workspace/up/detach_test.go index 7decfb8189..3af0869600 100644 --- a/cmd/workspace/up/detach_test.go +++ b/cmd/workspace/up/detach_test.go @@ -53,6 +53,9 @@ func TestDetachWorkspaceLabelNormalizesPath(t *testing.T) { } func TestOpenTaskExitsWhenAlreadyCanceled(t *testing.T) { + // DEVSY_HOME redirects the task store on every platform; $HOME alone + // misses the Windows path manager, which reads %LOCALAPPDATA%. + t.Setenv("DEVSY_HOME", t.TempDir()) t.Setenv("HOME", t.TempDir()) config.ResetPathManager() t.Cleanup(config.ResetPathManager) From 132bd4077ae4f6262fe332a8a9a2c1a06514cbdf Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 09:21:29 -0700 Subject: [PATCH 12/49] test(e2e): poll for the tunnel SSH config after ready The ready phase can win the race against the config write on slower runners, so the Windows ssh-tunnel-mode spec read the file before the worker wrote it. --- e2e/tests/ssh/ssh_tunnel_mode.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/e2e/tests/ssh/ssh_tunnel_mode.go b/e2e/tests/ssh/ssh_tunnel_mode.go index 4203ba854a..437e89c5df 100644 --- a/e2e/tests/ssh/ssh_tunnel_mode.go +++ b/e2e/tests/ssh/ssh_tunnel_mode.go @@ -15,6 +15,19 @@ import ( const tunnelActiveTimeout = 4 * time.Minute +// readTunnelSSHConfig waits for the worker to write the SSH config: the +// ready phase can win the race against the config write on slower runners. +func readTunnelSSHConfig(ctx context.Context, path string) string { + var configBytes []byte + gomega.Eventually(func() error { + var err error + configBytes, err = os.ReadFile(filepath.Clean(path)) + return err + }).WithContext(ctx).WithTimeout(time.Minute).WithPolling(2 * time.Second). + Should(gomega.Succeed()) + return string(configBytes) +} + var _ = ginkgo.Describe( "devsy ssh tunnel mode", ginkgo.Label("ssh-tunnel-mode"), @@ -80,9 +93,7 @@ var _ = ginkgo.Describe( waitDetachedTunnelReady(ctx, f, taskID) - configBytes, err := os.ReadFile(filepath.Clean(sshConfigPath)) - framework.ExpectNoError(err) - config := string(configBytes) + config := readTunnelSSHConfig(ctx, sshConfigPath) gomega.Expect(config).To( gomega.ContainSubstring("Hostname 127.0.0.1"), @@ -123,9 +134,7 @@ var _ = ginkgo.Describe( waitDetachedTunnelReady(ctx, f, taskID) - configBytes, err := os.ReadFile(filepath.Clean(sshConfigPath)) - framework.ExpectNoError(err) - config := string(configBytes) + config := readTunnelSSHConfig(ctx, sshConfigPath) var port string for line := range strings.SplitSeq(config, "\n") { From 71cc8abd94cc0057bc71303698c495a660723286 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 11:03:27 -0600 Subject: [PATCH 13/49] fix(task): serialize cancellation with worker startup --- pkg/task/store.go | 32 ++++++++++++++++------------ pkg/task/task.go | 41 +++++++++++++++++++++++++++++------- pkg/task/task_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 20 deletions(-) diff --git a/pkg/task/store.go b/pkg/task/store.go index 1f5c53010d..21e97d35b2 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -29,6 +29,8 @@ type Store struct { killProcess func(pid, treeName string) error // Test seam; see Store.SetAfterClaimForTest. afterClaimForTest func() + // Test seam for the cancellation/worker-start lock handoff. + afterCancelLockClaimedForTest func() } func NewStore() (*Store, error) { @@ -210,6 +212,15 @@ func (s *Store) SetKillProcessForTest(fn func(pid, treeName string) error) { s.killProcess = fn } +// SetAfterCancelLockClaimedForTest pauses Cancel while it owns the worker +// lock, allowing tests to verify that a starting worker cannot pass it. +// +// Not in export_test.go: other packages' tests need it, and a _test.go file +// compiles only into its own package's test binary. +func (s *Store) SetAfterCancelLockClaimedForTest(fn func()) { + s.afterCancelLockClaimedForTest = fn +} + // awaitPID polls for the worker to publish its PID, which it does right // after claiming the worker lock. func (s *Store) awaitPID(id string, timeout time.Duration) int { @@ -257,27 +268,22 @@ func (s *Store) workerAlive(id string) bool { return false } -// workerStarting reports whether a live worker holds the task's worker lock -// without having published its PID yet: Cancel then has nothing to signal -// until the worker publishes. -func (s *Store) workerStarting(id string) bool { +// tryWorkerLock claims the task's worker lock when no worker owns it. Unlike +// claimDeadWorkerLock, it creates the lock file if the worker has not started. +func (s *Store) tryWorkerLock(id string) (*flock.Flock, bool, error) { path, err := s.workerLockPath(id) if err != nil { - return false - } - if _, err := os.Stat(path); err != nil { - return false + return nil, false, err } lock := flock.New(path) locked, err := lock.TryLock() if err != nil { - return false + return nil, false, err } - if locked { - _ = lock.Unlock() - return false + if !locked { + return nil, false, nil } - return true + return lock, true, nil } // claimDeadWorkerLock acquires the task's worker lock, which only succeeds when diff --git a/pkg/task/task.go b/pkg/task/task.go index 8aaf2da5e0..bb9db3fa78 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -182,13 +182,14 @@ func (t *Task) Cancel() error { return nil } - if pid == 0 && t.store.workerStarting(t.id) { - pid = t.store.awaitPID(t.id, pidPublishTimeout) - if pid == 0 { - return fmt.Errorf( - "cancel task %s: worker has not published its pid yet, retry", - t.id, - ) + if pid == 0 { + var finalized bool + pid, finalized, err = t.cancelWithoutPublishedPID() + if err != nil { + return err + } + if finalized { + return nil } } @@ -248,6 +249,32 @@ func (t *Task) Fail(err error) error { }) } +func (t *Task) cancelWithoutPublishedPID() (int, bool, error) { + workerLock, locked, err := t.store.tryWorkerLock(t.id) + if err != nil { + return 0, false, fmt.Errorf("cancel task %s: claim worker lock: %w", t.id, err) + } + if locked { + defer func() { _ = workerLock.Unlock() }() + if t.store.afterCancelLockClaimedForTest != nil { + t.store.afterCancelLockClaimedForTest() + } + // Holding the worker lock through the state update prevents a + // not-yet-started worker from passing its terminal-state check + // between our liveness check and cancellation commit. + return 0, true, t.store.update(t.id, markCanceled) + } + + pid := t.store.awaitPID(t.id, pidPublishTimeout) + if pid == 0 { + return 0, false, fmt.Errorf( + "cancel task %s: worker has not published its pid yet, retry", + t.id, + ) + } + return pid, false, nil +} + type taskReporter struct { task *Task } diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index aa16f3e404..95de2e9993 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -11,6 +11,7 @@ import ( "github.com/devsy-org/devsy/pkg/clierr" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/status" + "github.com/gofrs/flock" ) func newTestStore(t *testing.T) *Store { @@ -373,6 +374,54 @@ func TestCancelWithoutPIDMarksFailed(t *testing.T) { } } +func TestCancelHoldsWorkerLockThroughCancellationCommit(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + + lockClaimed := make(chan struct{}) + continueCancel := make(chan struct{}) + store.SetAfterCancelLockClaimedForTest(func() { + close(lockClaimed) + <-continueCancel + }) + cancelDone := make(chan error, 1) + go func() { cancelDone <- tk.Cancel() }() + <-lockClaimed + + path, err := store.workerLockPath(tk.ID()) + if err != nil { + t.Fatalf("workerLockPath: %v", err) + } + workerLock := flock.New(path) + assertWorkerLock(t, workerLock, false, "while Cancel owns worker lock") + + close(continueCancel) + if err := <-cancelDone; err != nil { + t.Fatalf("Cancel: %v", err) + } + assertWorkerLock(t, workerLock, true, "after Cancel") + defer func() { _ = workerLock.Unlock() }() + + requireCanceledState(t, store, tk.ID()) +} + +func assertWorkerLock(t *testing.T, lock *flock.Flock, wantLocked bool, when string) { + t.Helper() + locked, err := lock.TryLock() + if err != nil { + t.Fatalf("TryLock %s: %v", when, err) + } + if locked != wantLocked { + if locked { + _ = lock.Unlock() + } + t.Fatalf("worker lock held %t %s, want %t", locked, when, wantLocked) + } +} + func TestCancelOnTerminalTaskIsNoop(t *testing.T) { store := newTestStore(t) task, err := store.Create(CreateOptions{}) From 3d388b589b534ba10907737d4119e3eb467afdd7 Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 11:16:27 -0700 Subject: [PATCH 14/49] test(e2e): harden tunnel specs against cold-start races The task phase mixes sub-pipeline events, so a ready observation can precede the tunnel listener. Poll the TCP dial and give the first SSH command a retry window; later commands keep the warm-path budget. --- e2e/tests/ssh/ssh_tunnel_mode.go | 43 ++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/e2e/tests/ssh/ssh_tunnel_mode.go b/e2e/tests/ssh/ssh_tunnel_mode.go index 437e89c5df..fbec59ee04 100644 --- a/e2e/tests/ssh/ssh_tunnel_mode.go +++ b/e2e/tests/ssh/ssh_tunnel_mode.go @@ -147,11 +147,14 @@ var _ = ginkgo.Describe( gomega.Expect(port).NotTo(gomega.BeEmpty(), "should find Port in SSH config") addr := net.JoinHostPort("127.0.0.1", port) - conn, err := net.DialTimeout("tcp", addr, 5*time.Second) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - "should be able to connect to local tunnel port", - ) - _ = conn.Close() + gomega.Eventually(func() error { + conn, err := net.DialTimeout("tcp", addr, 5*time.Second) + if err != nil { + return err + } + return conn.Close() + }).WithContext(ctx).WithTimeout(time.Minute).WithPolling(2*time.Second). + Should(gomega.Succeed(), "should be able to connect to local tunnel port") }, ) @@ -176,14 +179,28 @@ var _ = ginkgo.Describe( waitDetachedTunnelReady(ctx, f, taskID) - for i := range 3 { - sshCtx, cancelSSH := context.WithDeadline(ctx, time.Now().Add(20*time.Second)) - out, err := f.DevsySSH( - sshCtx, - tempDir, - "echo iteration-"+strings.Repeat("x", i), - ) - cancelSSH() + runSSH := func(i int, budget time.Duration) (string, error) { + sshCtx, cancelSSH := context.WithDeadline(ctx, time.Now().Add(budget)) + defer cancelSSH() + return f.DevsySSH(sshCtx, tempDir, "echo iteration-"+strings.Repeat("x", i)) + } + + // The task phase mixes sub-pipeline events, so the first + // command can still race the tunnel listener on a cold + // container; later commands then exercise the warm path. + var out string + gomega.Eventually(func() error { + var err error + out, err = runSSH(0, time.Minute) + return err + }).WithContext(ctx).WithTimeout(3 * time.Minute).WithPolling(10 * time.Second). + Should(gomega.Succeed()) + gomega.Expect(out).To( + gomega.ContainSubstring("iteration-"), + "sequential SSH command should succeed", + ) + for i := 1; i < 3; i++ { + out, err := runSSH(i, 20*time.Second) framework.ExpectNoError(err) gomega.Expect(out).To( gomega.ContainSubstring("iteration-"), From cd361200b595dbb55241a6877b1d3a363fb033aa Mon Sep 17 00:00:00 2001 From: "devsy-app[bot]" <277138668+devsy-app[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 12:08:03 -0700 Subject: [PATCH 15/49] fix(task): treat terminal state during pid wait as quiesced A cancellation that loses the worker-lock race to a concurrent cancel waited the full pid-publish timeout for a pid that never comes, failing workspace stop or delete even though the task was already canceled. --- pkg/task/store.go | 16 +++++++++++----- pkg/task/task.go | 5 ++++- pkg/task/task_test.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/pkg/task/store.go b/pkg/task/store.go index 21e97d35b2..d55595640a 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -222,16 +222,22 @@ func (s *Store) SetAfterCancelLockClaimedForTest(fn func()) { } // awaitPID polls for the worker to publish its PID, which it does right -// after claiming the worker lock. -func (s *Store) awaitPID(id string, timeout time.Duration) int { +// after claiming the worker lock. A concurrent cancellation can commit the +// terminal state first, leaving nothing to wait for. +func (s *Store) awaitPID(id string, timeout time.Duration) (int, bool) { deadline := time.Now().Add(timeout) for { state, err := s.Get(id) - if err == nil && state.PID != 0 { - return state.PID + if err == nil { + if state.PID != 0 { + return state.PID, false + } + if state.Status.Terminal() { + return 0, true + } } if time.Now().After(deadline) { - return 0 + return 0, false } time.Sleep(50 * time.Millisecond) } diff --git a/pkg/task/task.go b/pkg/task/task.go index bb9db3fa78..2a35faf229 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -265,7 +265,10 @@ func (t *Task) cancelWithoutPublishedPID() (int, bool, error) { return 0, true, t.store.update(t.id, markCanceled) } - pid := t.store.awaitPID(t.id, pidPublishTimeout) + pid, terminal := t.store.awaitPID(t.id, pidPublishTimeout) + if terminal { + return 0, true, nil + } if pid == 0 { return 0, false, fmt.Errorf( "cancel task %s: worker has not published its pid yet, retry", diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index 95de2e9993..28ec155e82 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -422,6 +422,37 @@ func assertWorkerLock(t *testing.T, lock *flock.Flock, wantLocked bool, when str } } +func TestOverlappingCancelsBeforePIDBothSucceed(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + + lockClaimed := make(chan struct{}) + continueCancel := make(chan struct{}) + store.SetAfterCancelLockClaimedForTest(func() { + close(lockClaimed) + <-continueCancel + }) + firstDone := make(chan error, 1) + go func() { firstDone <- tk.Cancel() }() + <-lockClaimed + + secondDone := make(chan error, 1) + go func() { secondDone <- tk.Cancel() }() + time.Sleep(200 * time.Millisecond) + close(continueCancel) + + if err := <-firstDone; err != nil { + t.Fatalf("first Cancel: %v", err) + } + if err := <-secondDone; err != nil { + t.Fatalf("second Cancel: %v", err) + } + requireCanceledState(t, store, tk.ID()) +} + func TestCancelOnTerminalTaskIsNoop(t *testing.T) { store := newTestStore(t) task, err := store.Create(CreateOptions{}) From 27aa6a4c971472a16ba43cb7210ef0dada23f2be Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 16:25:14 -0600 Subject: [PATCH 16/49] fix(task): close remaining cancellation lifecycle races --- cmd/workspace/stop_test.go | 1 + cmd/workspace/up/detach.go | 4 +- cmd/workspace/up/detach_test.go | 14 ++++ pkg/command/background.go | 18 ++-- pkg/command/job_windows.go | 50 +++++++++++- pkg/command/process_supported.go | 4 + pkg/command/process_windows.go | 26 +++--- pkg/command/process_windows_test.go | 122 ++++++++++++++++++++++++++-- 8 files changed, 210 insertions(+), 29 deletions(-) diff --git a/cmd/workspace/stop_test.go b/cmd/workspace/stop_test.go index 4afd76ae30..c95bb0c789 100644 --- a/cmd/workspace/stop_test.go +++ b/cmd/workspace/stop_test.go @@ -91,6 +91,7 @@ func TestStopDoesNotProceedWhenActiveTaskCannotBeKilled(t *testing.T) { func TestStopFindsTaskWithoutDesktopState(t *testing.T) { home := t.TempDir() + t.Setenv("DEVSY_HOME", home) t.Setenv("HOME", home) t.Setenv("USERPROFILE", home) config.ResetPathManager() diff --git a/cmd/workspace/up/detach.go b/cmd/workspace/up/detach.go index 7c8b2a492d..85ca291566 100644 --- a/cmd/workspace/up/detach.go +++ b/cmd/workspace/up/detach.go @@ -8,6 +8,7 @@ import ( "github.com/devsy-org/devsy/pkg/command" config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/file" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/output" "github.com/devsy-org/devsy/pkg/task" @@ -85,7 +86,8 @@ func (cmd *UpCmd) detachWorkspaceLabel(args []string) string { return cmd.ID } if len(args) > 0 { - return workspace2.ToID(args[0]) + _, source := file.IsLocalDir(args[0]) + return workspace2.ToID(source) } return "" } diff --git a/cmd/workspace/up/detach_test.go b/cmd/workspace/up/detach_test.go index 3af0869600..fec8732072 100644 --- a/cmd/workspace/up/detach_test.go +++ b/cmd/workspace/up/detach_test.go @@ -2,6 +2,7 @@ package up import ( "errors" + "os" "reflect" "testing" @@ -52,6 +53,19 @@ func TestDetachWorkspaceLabelNormalizesPath(t *testing.T) { } } +func TestDetachWorkspaceLabelResolvesCurrentDirectory(t *testing.T) { + cmd := &UpCmd{} + got := cmd.detachWorkspaceLabel([]string{"."}) + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("Getwd: %v", err) + } + want := workspace2.ToID(cwd) + if got != want { + t.Fatalf("detachWorkspaceLabel(.) = %q, want %q", got, want) + } +} + func TestOpenTaskExitsWhenAlreadyCanceled(t *testing.T) { // DEVSY_HOME redirects the task store on every platform; $HOME alone // misses the Windows path manager, which reads %LOCALAPPDATA%. diff --git a/pkg/command/background.go b/pkg/command/background.go index 3b39d62670..383ea24ba6 100644 --- a/pkg/command/background.go +++ b/pkg/command/background.go @@ -119,12 +119,9 @@ func startDetached(cmd *exec.Cmd, commandName, pidFile, streamsFile string) erro closeFile(streamsF) if err := ownProcessTree(cmd.Process.Pid, commandName); err != nil { - fmt.Fprintf( - os.Stderr, - "warning: process tree ownership unavailable for pid %d: %v\n", - cmd.Process.Pid, - err, - ) + _ = cmd.Process.Kill() + _ = cmd.Wait() + return fmt.Errorf("assign process tree ownership for pid %d: %w", cmd.Process.Pid, err) } if pidFile != "" { @@ -135,6 +132,15 @@ func startDetached(cmd *exec.Cmd, commandName, pidFile, streamsFile string) erro } } + if err := resumeBackgroundTree(cmd.Process.Pid); err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + if pidFile != "" { + _ = os.Remove(pidFile) + } + return fmt.Errorf("resume process %d: %w", cmd.Process.Pid, err) + } + _ = cmd.Process.Release() return nil diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index a43e0495e8..d981207add 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -5,6 +5,8 @@ package command import ( "errors" "fmt" + "os/exec" + "syscall" "unsafe" "golang.org/x/sys/windows" @@ -20,10 +22,54 @@ func jobNameFor(name string) string { return name } +func prepareBackgroundTree(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.CreationFlags |= windows.CREATE_SUSPENDED +} + +// resumeBackgroundTree resumes the primary thread of a process created with +// CREATE_SUSPENDED. That process cannot spawn descendants until it has been +// assigned to its Job Object. +func resumeBackgroundTree(pid int) error { + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) + if err != nil { + return fmt.Errorf("snapshot process threads: %w", err) + } + defer func() { _ = windows.CloseHandle(snapshot) }() + + entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))} + if err := windows.Thread32First(snapshot, &entry); err != nil { + return fmt.Errorf("find primary thread for process %d: %w", pid, err) + } + for { + if entry.OwnerProcessID == uint32(pid) { + thread, err := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID) + if err != nil { + return fmt.Errorf("open primary thread %d: %w", entry.ThreadID, err) + } + defer func() { _ = windows.CloseHandle(thread) }() + + previous, err := windows.ResumeThread(thread) + if err != nil { + return fmt.Errorf("resume primary thread %d: %w", entry.ThreadID, err) + } + if previous != 1 { + return fmt.Errorf("primary thread %d suspend count was %d, want 1", entry.ThreadID, previous) + } + return nil + } + if err := windows.Thread32Next(snapshot, &entry); err != nil { + return fmt.Errorf("find primary thread for process %d: %w", pid, err) + } + } +} + // ownProcessTree assigns the process to a Job Object named for the worker. // Job membership outlives intermediate parents, so tree teardown stays -// complete where taskkill /T loses track. Assignment failure is non-fatal; -// termination then falls back to taskkill /T /F. +// complete where taskkill /T loses track. Assignment must succeed before the +// suspended worker is resumed so descendants cannot escape the job. func ownProcessTree(pid int, workerName string) error { name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) if err != nil { diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index 955adfd4d6..0dded2e1d8 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -63,6 +63,10 @@ func ownProcessTree(pid int, name string) error { return nil } +func resumeBackgroundTree(pid int) error { + return nil +} + func prepareBackgroundTree(cmd *exec.Cmd) { if cmd.SysProcAttr == nil { cmd.SysProcAttr = &syscall.SysProcAttr{} diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index 0494f3de43..682319fde0 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -12,20 +12,13 @@ import ( "golang.org/x/sys/windows" ) -// stillActive is the exit code GetExitCodeProcess reports for a live process. -const stillActive uint32 = 259 - func isRunning(pid string) (bool, error) { parsed, err := parsePID(pid) if err != nil { return false, err } - handle, err := windows.OpenProcess( - windows.PROCESS_QUERY_LIMITED_INFORMATION, - false, - uint32(parsed), - ) + handle, err := windows.OpenProcess(windows.SYNCHRONIZE, false, uint32(parsed)) if err != nil { // OpenProcess reports a nonexistent PID as ERROR_INVALID_PARAMETER; // anything else (e.g. access denied) is a real query failure. @@ -36,11 +29,18 @@ func isRunning(pid string) (bool, error) { } defer func() { _ = windows.CloseHandle(handle) }() - var exitCode uint32 - if err := windows.GetExitCodeProcess(handle, &exitCode); err != nil { - return false, fmt.Errorf("query process %d: %w", parsed, err) + event, err := windows.WaitForSingleObject(handle, 0) + if err != nil { + return false, fmt.Errorf("wait for process %d: %w", parsed, err) + } + switch event { + case windows.WAIT_OBJECT_0: + return false, nil + case uint32(windows.WAIT_TIMEOUT): + return true, nil + default: + return false, fmt.Errorf("wait for process %d returned unexpected result %#x", parsed, event) } - return exitCode == stillActive, nil } func killTree(pid, treeName string) error { @@ -86,8 +86,6 @@ func killTree(pid, treeName string) error { return verifyTerminated(parsed) } -func prepareBackgroundTree(*exec.Cmd) {} - // verifyTerminated allows a brief window for the exit to become visible // after the termination call reported success. func verifyTerminated(parsed int) error { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index ce0680f830..3ea0afc7ec 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -10,6 +10,9 @@ import ( "strings" "testing" "time" + "unsafe" + + "golang.org/x/sys/windows" ) const ( @@ -18,14 +21,34 @@ const ( helperEnvChildPIDFile = "DEVSY_TEST_HELPER_CHILD_PIDFILE" helperEnvWaitFile = "DEVSY_TEST_HELPER_WAIT_FILE" helperEnvExitAfterSpawn = "DEVSY_TEST_HELPER_EXIT_AFTER_SPAWN" + helperEnvExitCode = "DEVSY_TEST_HELPER_EXIT_CODE" + helperEnvCheckJob = "DEVSY_TEST_HELPER_CHECK_JOB" ) +var procIsProcessInJob = windows.NewLazySystemDLL("kernel32.dll").NewProc("IsProcessInJob") + // TestHelperProcess is not a test; every helper process re-executes the test // binary into it. func TestHelperProcess(t *testing.T) { if os.Getenv(helperEnvMarker) != "1" { return } + if exitCode := os.Getenv(helperEnvExitCode); exitCode != "" { + code, err := strconv.Atoi(exitCode) + if err != nil { + os.Exit(2) + } + os.Exit(code) + } + if jobResult := os.Getenv(helperEnvCheckJob); jobResult != "" { + inJob, err := currentProcessInJob() + if err != nil { + os.Exit(3) + } + if err := os.WriteFile(jobResult, []byte(strconv.FormatBool(inJob)), 0o600); err != nil { + os.Exit(4) + } + } if waitFile := os.Getenv(helperEnvWaitFile); waitFile != "" { for { if _, err := os.Stat(waitFile); err == nil { @@ -125,6 +148,88 @@ func TestIsRunningDetectsLiveAndExitedProcess(t *testing.T) { assertNotRunningEventually(t, cmd.Process.Pid) } +func TestIsRunningRecognizesExitCode259AsExited(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run", "TestHelperProcess") + cmd.Env = helperEnv(helperEnvMarker+"=1", helperEnvExitCode+"=259") + if err := cmd.Start(); err != nil { + t.Fatalf("start helper: %v", err) + } + processHandle, err := windows.OpenProcess(windows.SYNCHRONIZE, false, uint32(cmd.Process.Pid)) + if err != nil { + _ = cmd.Process.Kill() + _ = cmd.Wait() + t.Fatalf("open helper process: %v", err) + } + defer func() { _ = windows.CloseHandle(processHandle) }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + + result, err := windows.WaitForSingleObject(processHandle, 10_000) + if err != nil { + t.Fatalf("wait for helper: %v", err) + } + if result != windows.WAIT_OBJECT_0 { + t.Fatalf("wait result = %#x, want WAIT_OBJECT_0", result) + } + + running, err := isRunning(strconv.Itoa(cmd.Process.Pid)) + if err != nil { + t.Fatalf("isRunning(%d): %v", cmd.Process.Pid, err) + } + if running { + t.Fatal("isRunning returned true for exited process with exit code 259") + } + + if err := cmd.Wait(); err == nil || cmd.ProcessState.ExitCode() != 259 { + t.Fatalf("helper exit = %v (%v), want exit code 259", err, cmd.ProcessState) + } +} + +func TestStartDetachedAssignsWorkerToJobBeforeItRuns(t *testing.T) { + dir := t.TempDir() + jobName := "devsy-test-suspended-worker" + jobResult := filepath.Join(dir, "in-job") + pidFile := filepath.Join(dir, "worker.pid") + cmd := exec.Command(os.Args[0], "-test.run", "TestHelperProcess") + cmd.Env = helperEnv(helperEnvMarker+"=1", helperEnvCheckJob+"="+jobResult) + + if err := startDetached(cmd, jobName, pidFile, filepath.Join(dir, "streams")); err != nil { + t.Fatalf("startDetached: %v", err) + } + pid := waitForPIDFile(t, pidFile) + t.Cleanup(func() { _ = KillTree(strconv.Itoa(pid), jobName) }) + + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(jobResult); err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + data, err := os.ReadFile(jobResult) // #nosec G304 -- test-controlled path + if err != nil { + t.Fatalf("read job membership result: %v", err) + } + if strings.TrimSpace(string(data)) != "true" { + t.Fatalf("worker job membership = %q, want true", data) + } +} + +func currentProcessInJob() (bool, error) { + var inJob int32 + result, _, callErr := procIsProcessInJob.Call( + uintptr(windows.CurrentProcess()), + 0, + uintptr(unsafe.Pointer(&inJob)), + ) + if result == 0 { + return false, callErr + } + return inJob != 0, nil +} + func waitForPIDFile(t *testing.T, pidFile string) int { t.Helper() deadline := time.Now().Add(10 * time.Second) @@ -203,14 +308,19 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { // The child spawns the grandchild and exits, orphaning the grandchild // from the perspective of parent-based termination. - parent := startHelper(t, + parentPIDFile := filepath.Join(dir, "parent.pid") + parent := exec.Command(os.Args[0], "-test.run", "TestHelperProcess") + parent.Env = helperEnv( + helperEnvMarker+"=1", helperEnvWaitFile+"="+waitFile, helperEnvPIDFile+"="+childPIDFile, helperEnvChildPIDFile+"="+grandchildPIDFile, ) - if err := ownProcessTree(parent.Process.Pid, "devsy-test-worker"); err != nil { - t.Skipf("job assignment unavailable in this environment: %v", err) + if err := startDetached(parent, "devsy-test-worker", parentPIDFile, filepath.Join(dir, "streams")); err != nil { + t.Fatalf("startDetached: %v", err) } + parentPID := waitForPIDFile(t, parentPIDFile) + t.Cleanup(func() { _ = KillTree(strconv.Itoa(parentPID), "devsy-test-worker") }) if err := os.WriteFile(waitFile, []byte("go"), 0o600); err != nil { t.Fatalf("signal helper: %v", err) } @@ -218,13 +328,13 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { childPID := waitForPIDFile(t, childPIDFile) grandchildPID := waitForPIDFile(t, grandchildPIDFile) assertNotRunningEventually(t, childPID) - assertRunning(t, parent.Process.Pid) + assertRunning(t, parentPID) assertRunning(t, grandchildPID) - if err := KillTree(strconv.Itoa(parent.Process.Pid), "devsy-test-worker"); err != nil { + if err := KillTree(strconv.Itoa(parentPID), "devsy-test-worker"); err != nil { t.Fatalf("Kill(parent): %v", err) } - assertNotRunningEventually(t, parent.Process.Pid) + assertNotRunningEventually(t, parentPID) assertNotRunningEventually(t, grandchildPID) } From be1a2dc69e3042ad1b1e148ab113eac6e9751144 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 17:02:59 -0600 Subject: [PATCH 17/49] style(task): apply golangci-lint formatting --- pkg/command/job_windows.go | 6 +++++- pkg/command/process_windows.go | 6 +++++- pkg/command/process_windows_test.go | 7 ++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index d981207add..3be3d18f16 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -56,7 +56,11 @@ func resumeBackgroundTree(pid int) error { return fmt.Errorf("resume primary thread %d: %w", entry.ThreadID, err) } if previous != 1 { - return fmt.Errorf("primary thread %d suspend count was %d, want 1", entry.ThreadID, previous) + return fmt.Errorf( + "primary thread %d suspend count was %d, want 1", + entry.ThreadID, + previous, + ) } return nil } diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index 682319fde0..cd3d88abc2 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -39,7 +39,11 @@ func isRunning(pid string) (bool, error) { case uint32(windows.WAIT_TIMEOUT): return true, nil default: - return false, fmt.Errorf("wait for process %d returned unexpected result %#x", parsed, event) + return false, fmt.Errorf( + "wait for process %d returned unexpected result %#x", + parsed, + event, + ) } } diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 3ea0afc7ec..2b4b39b217 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -316,7 +316,12 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { helperEnvPIDFile+"="+childPIDFile, helperEnvChildPIDFile+"="+grandchildPIDFile, ) - if err := startDetached(parent, "devsy-test-worker", parentPIDFile, filepath.Join(dir, "streams")); err != nil { + if err := startDetached( + parent, + "devsy-test-worker", + parentPIDFile, + filepath.Join(dir, "streams"), + ); err != nil { t.Fatalf("startDetached: %v", err) } parentPID := waitForPIDFile(t, parentPIDFile) From 3ad8efb2ee8fea0c12187606079242467d19652f Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 18:44:09 -0600 Subject: [PATCH 18/49] fix(task): verify process tree identity during cancellation --- cmd/workspace/up/detach.go | 5 + cmd/workspace/up/detach_test.go | 12 ++ pkg/command/job_windows.go | 102 +++++++++++- pkg/command/process.go | 15 ++ pkg/command/process_identity_darwin.go | 121 ++++++++++++++ pkg/command/process_identity_linux.go | 96 ++++++++++++ pkg/command/process_identity_unix.go | 13 ++ pkg/command/process_supported.go | 143 +++++++++++++++-- pkg/command/process_test.go | 209 +++++++++++++++++++++++++ pkg/command/process_windows.go | 89 ++++++++++- pkg/command/process_windows_test.go | 34 +++- pkg/task/store.go | 53 +++++-- pkg/task/store_test.go | 60 +++++++ pkg/task/task.go | 129 ++++++++++----- pkg/task/task_pid_supported_test.go | 50 ++++++ pkg/task/task_test.go | 82 +++++++++- 16 files changed, 1132 insertions(+), 81 deletions(-) create mode 100644 pkg/command/process_identity_darwin.go create mode 100644 pkg/command/process_identity_linux.go create mode 100644 pkg/command/process_identity_unix.go diff --git a/cmd/workspace/up/detach.go b/cmd/workspace/up/detach.go index 85ca291566..7aa1465e83 100644 --- a/cmd/workspace/up/detach.go +++ b/cmd/workspace/up/detach.go @@ -117,16 +117,21 @@ func (cmd *UpCmd) openTask() (*task.Task, error) { } if err := t.SetPID(os.Getpid()); err != nil { failTask(t, err) + _ = t.ReleaseWorkerLock() return nil, err } state, err := store.Get(cmd.taskID) if err != nil { failTask(t, err) + _ = t.ReleaseWorkerLock() return nil, err } // Canceled before this worker claimed its lock: the canceled state is // already recorded and this worker must not overwrite it or run up. if state.Status.Terminal() { + if err := t.ReleaseWorkerLock(); err != nil { + return nil, fmt.Errorf("release canceled worker lock: %w", err) + } return nil, task.ErrCanceled } return t, nil diff --git a/cmd/workspace/up/detach_test.go b/cmd/workspace/up/detach_test.go index fec8732072..230ad72f68 100644 --- a/cmd/workspace/up/detach_test.go +++ b/cmd/workspace/up/detach_test.go @@ -99,4 +99,16 @@ func TestOpenTaskExitsWhenAlreadyCanceled(t *testing.T) { if state.Error != task.ErrCanceled.Error() { t.Errorf("openTask overwrote the canceled state: %+v", state) } + assertWorkerLockReleased(t, store, tk.ID()) +} + +func assertWorkerLockReleased(t *testing.T, store *task.Store, id string) { + t.Helper() + lockHolder := store.Open(id) + if err := lockHolder.HoldWorkerLock(); err != nil { + t.Fatalf("worker lock remained held after canceled startup: %v", err) + } + if err := lockHolder.ReleaseWorkerLockForTest(); err != nil { + t.Fatalf("release worker lock: %v", err) + } } diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index 3be3d18f16..84a575b59b 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -7,6 +7,7 @@ import ( "fmt" "os/exec" "syscall" + "time" "unsafe" "golang.org/x/sys/windows" @@ -14,7 +15,23 @@ import ( const jobObjectTerminateAccess = 0x0008 +const jobObjectQueryAccess = 0x0004 + +const jobTerminationTimeout = 5 * time.Second + +type jobAccountingInfo struct { + totalUserTime int64 + totalKernelTime int64 + thisPeriodTotalUserTime int64 + thisPeriodTotalKernelTime int64 + totalPageFaultCount uint32 + totalProcesses uint32 + activeProcesses uint32 + totalTerminatedProcesses uint32 +} + var procOpenJobObjectW = windows.NewLazySystemDLL("kernel32.dll").NewProc("OpenJobObjectW") +var procIsProcessInJobForTermination = windows.NewLazySystemDLL("kernel32.dll").NewProc("IsProcessInJob") // jobNameFor names the Job Object after the worker (e.g. devsy-up-) // rather than its PID: a reused PID must never inherit a predecessor's job. @@ -105,26 +122,97 @@ func ownProcessTree(pid int, workerName string) error { // reports whether a job existed; without one the caller falls back to // taskkill, e.g. for workers launched before job ownership existed. func terminateJob(workerName string) (bool, error) { - name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) + job, found, err := openWorkerJob(workerName) + if err != nil || !found { + return found, err + } + defer func() { _ = windows.CloseHandle(job) }() + return terminateJobHandle(job, workerName) +} + +func terminateJobForPID(workerName string, pid int) (bool, error) { + job, found, err := openWorkerJob(workerName) + if err != nil || !found { + return found, err + } + defer func() { _ = windows.CloseHandle(job) }() + + inJob, err := processBelongsToJob(pid, job) + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + return false, fmt.Errorf("worker process %d exited before job membership was verified", pid) + } if err != nil { return false, err } + if !inJob { + return false, fmt.Errorf("process %d does not belong to worker job %s", pid, workerName) + } + return terminateJobHandle(job, workerName) +} + +func processBelongsToJob(pid int, job windows.Handle) (bool, error) { + process, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err != nil { + return false, fmt.Errorf("open worker process %d: %w", pid, err) + } + defer func() { _ = windows.CloseHandle(process) }() + var inJob int32 + result, _, callErr := procIsProcessInJobForTermination.Call( + uintptr(process), + uintptr(job), + uintptr(unsafe.Pointer(&inJob)), + ) + if result == 0 { + return false, fmt.Errorf("verify process %d in job: %w", pid, callErr) + } + return inJob != 0, nil +} + +func openWorkerJob(workerName string) (windows.Handle, bool, error) { + name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) + if err != nil { + return 0, false, err + } handle, _, callErr := procOpenJobObjectW.Call( - uintptr(jobObjectTerminateAccess), + uintptr(jobObjectTerminateAccess|jobObjectQueryAccess), 0, uintptr(unsafe.Pointer(name)), ) if handle == 0 { if errors.Is(callErr, windows.ERROR_FILE_NOT_FOUND) { - return false, nil + return 0, false, nil } - return false, fmt.Errorf("open job %s: %w", workerName, callErr) + return 0, false, fmt.Errorf("open job %s: %w", workerName, callErr) } - job := windows.Handle(handle) - defer func() { _ = windows.CloseHandle(job) }() + return windows.Handle(handle), true, nil +} +func terminateJobHandle(job windows.Handle, workerName string) (bool, error) { if err := windows.TerminateJobObject(job, 1); err != nil { return false, fmt.Errorf("terminate job %s: %w", workerName, err) } - return true, nil + deadline := time.Now().Add(jobTerminationTimeout) + for { + var info jobAccountingInfo + if err := windows.QueryInformationJobObject( + job, + windows.JobObjectBasicAccountingInformation, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)), + nil, + ); err != nil { + return false, fmt.Errorf("query job %s processes: %w", workerName, err) + } + if info.activeProcesses == 0 { + return true, nil + } + if time.Now().After(deadline) { + return false, fmt.Errorf( + "job %s still has %d active processes after termination", + workerName, + info.activeProcesses, + ) + } + time.Sleep(50 * time.Millisecond) + } } diff --git a/pkg/command/process.go b/pkg/command/process.go index fb1d422594..da16d63bf2 100644 --- a/pkg/command/process.go +++ b/pkg/command/process.go @@ -13,3 +13,18 @@ func Kill(pid string) error { func KillTree(pid, treeName string) error { return killTree(pid, treeName) } + +// KillTreeWithIdentity terminates a worker tree using its saved platform identity. +func KillTreeWithIdentity(pid, treeName, identity string) error { + return killTreeWithIdentity(pid, treeName, identity) +} + +// KillTreeAfterWorkerExit terminates descendants using the worker's saved session identity. +func KillTreeAfterWorkerExit(pid, treeName, identity string) error { + return killTreeAfterWorkerExit(pid, treeName, identity) +} + +// ProcessTreeIdentity returns the kernel identity used to distinguish a reused process ID. +func ProcessTreeIdentity(pid int) (string, error) { + return processTreeIdentity(pid) +} diff --git a/pkg/command/process_identity_darwin.go b/pkg/command/process_identity_darwin.go new file mode 100644 index 0000000000..ff317d9a9e --- /dev/null +++ b/pkg/command/process_identity_darwin.go @@ -0,0 +1,121 @@ +//go:build darwin + +package command + +import ( + "errors" + "fmt" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/unix" +) + +func processTreeIdentity(pid int) (string, error) { + sessionID, err := darwinSessionID(pid) + if err != nil { + if processLookupConfirmsExit(pid, err) { + return "", nil + } + return "", err + } + process, err := unix.SysctlKinfoProc("kern.proc.pid", pid) + if err != nil { + if processLookupConfirmsExit(pid, err) { + return "", nil + } + return "", err + } + return fmt.Sprintf( + "%d:%d:%d", + sessionID, + process.Proc.P_starttime.Sec, + process.Proc.P_starttime.Usec, + ), nil +} + +func processLookupConfirmsExit(pid int, lookupErr error) bool { + if !errors.Is(lookupErr, syscall.ESRCH) && !errors.Is(lookupErr, syscall.EIO) { + return false + } + return errors.Is(syscall.Kill(pid, 0), syscall.ESRCH) +} + +func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { + identityParts := strings.Split(identity, ":") + if len(identityParts) != 3 { + return false, fmt.Errorf("parse process tree identity %q", identity) + } + wantSession, err := strconv.Atoi(identityParts[0]) + if err != nil { + return false, fmt.Errorf("parse session identity %q: %w", identity, err) + } + for _, part := range identityParts[1:] { + if _, err := strconv.ParseInt(part, 10, 64); err != nil { + return false, fmt.Errorf("parse process start identity %q: %w", identity, err) + } + } + processes, err := unix.SysctlKinfoProcSlice("kern.proc.all") + if err != nil { + return false, err + } + return darwinProcessGroupHasSession(processes, pgid, wantSession, identity) +} + +func darwinProcessGroupHasSession( + processes []unix.KinfoProc, + pgid, wantSession int, + identity string, +) (bool, error) { + foundMember := false + for _, process := range processes { + matches, leaderMismatch, err := darwinProcessGroupMemberMatches( + process, + pgid, + wantSession, + identity, + ) + if err != nil { + return false, err + } + if leaderMismatch { + return false, nil + } + foundMember = foundMember || matches + } + return foundMember, nil +} + +func darwinProcessGroupMemberMatches( + process unix.KinfoProc, + pgid, wantSession int, + identity string, +) (matches, leaderMismatch bool, err error) { + processGroupID := int(process.Eproc.Pgid) + if processGroupID != pgid { + return false, false, nil + } + pid := int(process.Proc.P_pid) + sessionID, err := darwinSessionID(pid) + if errors.Is(err, syscall.ESRCH) { + return false, false, nil + } + if err != nil { + return false, false, err + } + if pid == pgid { + leaderIdentity, err := processTreeIdentity(pid) + if err != nil { + return false, false, err + } + if leaderIdentity != "" && leaderIdentity != identity { + return false, true, nil + } + } + return sessionID == wantSession, false, nil +} + +func darwinSessionID(pid int) (int, error) { + return unix.Getsid(pid) +} diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go new file mode 100644 index 0000000000..be801d3db7 --- /dev/null +++ b/pkg/command/process_identity_linux.go @@ -0,0 +1,96 @@ +//go:build linux + +package command + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +func processTreeIdentity(pid int) (string, error) { + stat, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", err + } + fields, ok := linuxProcessStatFields(stat) + if !ok || len(fields) < 20 { + return "", fmt.Errorf("read session identity for process %d", pid) + } + return fields[3] + ":" + fields[19], nil +} + +func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { + identityParts := strings.Split(identity, ":") + if len(identityParts) != 2 { + return false, fmt.Errorf("parse process tree identity %q", identity) + } + wantSession, err := strconv.Atoi(identityParts[0]) + if err != nil { + return false, fmt.Errorf("parse session identity %q: %w", identity, err) + } + wantStart, err := strconv.ParseUint(identityParts[1], 10, 64) + if err != nil { + return false, fmt.Errorf("parse process start identity %q: %w", identity, err) + } + entries, err := os.ReadDir("/proc") + if err != nil { + return false, err + } + + foundMember := false + // A live group leader must still have the saved start time as well as the session ID. + for _, entry := range entries { + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + processDir := filepath.Join("/proc", entry.Name()) + stat, err := os.ReadFile(filepath.Join(processDir, "stat")) + if err != nil { + if !os.IsNotExist(err) { + return false, err + } + continue + } + fields, ok := linuxProcessStatFields(stat) + if !ok || len(fields) < 4 { + continue + } + processGroup, err := strconv.Atoi(fields[2]) + if err != nil || processGroup != pgid { + continue + } + sessionID, err := strconv.Atoi(fields[3]) + if err != nil { + continue + } + if pid == pgid { + leaderStart, err := strconv.ParseUint(fields[19], 10, 64) + if err != nil { + return false, fmt.Errorf("parse process start time for process %d: %w", pid, err) + } + if sessionID != wantSession || leaderStart != wantStart { + return false, nil + } + } + if sessionID == wantSession { + foundMember = true + } + } + return foundMember, nil +} + +func linuxProcessStatFields(stat []byte) ([]string, bool) { + statText := string(stat) + closingParen := strings.LastIndexByte(statText, ')') + if closingParen < 0 || closingParen+2 >= len(statText) { + return nil, false + } + return strings.Fields(statText[closingParen+2:]), true +} diff --git a/pkg/command/process_identity_unix.go b/pkg/command/process_identity_unix.go new file mode 100644 index 0000000000..8252bed42b --- /dev/null +++ b/pkg/command/process_identity_unix.go @@ -0,0 +1,13 @@ +//go:build unix && !linux && !darwin + +package command + +import "errors" + +func processTreeIdentity(_ int) (string, error) { + return "", errors.New("process tree identity is unsupported on this platform") +} + +func processGroupMatchesIdentity(_ int, _ string) (bool, error) { + return false, errors.New("process tree identity is unsupported on this platform") +} diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index 0dded2e1d8..d42b3633f1 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -4,6 +4,7 @@ package command import ( "errors" + "fmt" "os" "os/exec" "strconv" @@ -30,33 +31,155 @@ func isRunning(pid string) (bool, error) { return true, nil } -// A background process that leads its own group (prepareBackgroundTree) is -// signaled group-wide so descendants that outlive it still die with it. -func killTree(pid, _ string) error { +// Detached workers use a session identity to distinguish their group after the leader exits. +func killTree(pid, treeName string) error { parsedPid, err := strconv.Atoi(pid) if err != nil { return err } + identity, err := processTreeIdentity(parsedPid) + if err != nil { + return fmt.Errorf("read process tree identity for worker %d: %w", parsedPid, err) + } + if identity == "" { + if treeName == "" { + return nil + } + return checkUnidentifiedProcessGroup(parsedPid, treeName) + } + return killTreeWithIdentity(pid, treeName, identity) +} + +func killTreeWithIdentity(pid, treeName, identity string) error { + parsedPid, err := strconv.Atoi(pid) + if err != nil { + return err + } + if parsedPid <= 0 { + return fmt.Errorf("invalid PID %d: must be positive", parsedPid) + } + if treeName == "" { + return killSingleProcessWithIdentity(pid, parsedPid, identity) + } + if identity == "" { + return errors.New("process tree identity is required to terminate a worker") + } + return killWorkerTreeWithIdentity(parsedPid, treeName, identity) +} + +func killSingleProcessWithIdentity(pid string, parsedPid int, identity string) error { + if identity == "" { + running, err := isRunning(pid) + if err != nil { + return err + } + if !running { + return nil + } + return errors.New("process identity is required to terminate a process") + } + return signalProcessTree(parsedPid, true, identity) +} + +func killWorkerTreeWithIdentity(parsedPid int, treeName, identity string) error { + groupMatches, err := processGroupMatchesIdentity(parsedPid, identity) + if err != nil { + return fmt.Errorf("verify process group %d for worker %s: %w", parsedPid, treeName, err) + } + if groupMatches { + return signalProcessTree(-parsedPid, false, identity) + } + currentIdentity, err := processTreeIdentity(parsedPid) + if err != nil { + return fmt.Errorf("read process tree identity for worker %d: %w", parsedPid, err) + } + // The saved group is empty, so a different identity at this PID is a reused process. + if currentIdentity != identity { + return nil + } + return signalProcessTree(parsedPid, false, identity) +} + +func killTreeAfterWorkerExit(pid, treeName, identity string) error { + parsedPid, err := strconv.Atoi(pid) + if err != nil { + return err + } + if parsedPid <= 0 { + return fmt.Errorf("invalid PID %d: must be positive", parsedPid) + } + if treeName == "" { + return errors.New("process tree name is required after worker exit") + } + if identity == "" { + return checkUnidentifiedProcessGroup(parsedPid, treeName) + } + matches, err := processGroupMatchesIdentity(parsedPid, identity) + if err != nil { + return fmt.Errorf( + "verify process group %d for worker tree %s: %w", + parsedPid, + treeName, + err, + ) + } + if !matches { + return nil + } + return signalProcessTree(-parsedPid, false, identity) +} - target := parsedPid - if pgid, err := syscall.Getpgid(parsedPid); err == nil && pgid == parsedPid { - target = -parsedPid +func checkUnidentifiedProcessGroup(pid int, treeName string) error { + if err := syscall.Kill(-pid, 0); errors.Is(err, syscall.ESRCH) { + return nil } + return fmt.Errorf("verify process group %d for worker tree %s", pid, treeName) +} +func signalProcessTree(target int, graceful bool, identity string) error { if err := syscall.Kill(target, syscall.SIGTERM); err != nil { if errors.Is(err, syscall.ESRCH) { return nil // already exited } - return err + return fmt.Errorf("send SIGTERM to process target %d: %w", target, err) + } + if graceful { + time.Sleep(2 * time.Second) + } + matches, err := signalTargetMatchesIdentity(target, identity) + if err != nil { + return fmt.Errorf("verify process target %d before SIGKILL: %w", target, err) + } + if !matches { + return nil } - time.Sleep(2 * time.Second) err = syscall.Kill(target, syscall.SIGKILL) if err != nil && !errors.Is(err, syscall.ESRCH) { - return err + return fmt.Errorf("send SIGKILL to process target %d: %w", target, err) } return nil } +func signalTargetMatchesIdentity(target int, identity string) (bool, error) { + if identity == "" { + return false, errors.New("process identity is required before SIGKILL") + } + if target < 0 { + return processGroupMatchesIdentity(-target, identity) + } + currentIdentity, err := processTreeIdentity(target) + if err != nil { + if errors.Is(err, syscall.ESRCH) { + return false, nil + } + if errors.Is(err, syscall.EIO) && errors.Is(syscall.Kill(target, 0), syscall.ESRCH) { + return false, nil + } + return false, err + } + return currentIdentity == identity, nil +} + // ownProcessTree is a no-op on Unix: the process group created by // prepareBackgroundTree already scopes the tree. func ownProcessTree(pid int, name string) error { @@ -71,5 +194,5 @@ func prepareBackgroundTree(cmd *exec.Cmd) { if cmd.SysProcAttr == nil { cmd.SysProcAttr = &syscall.SysProcAttr{} } - cmd.SysProcAttr.Setpgid = true + cmd.SysProcAttr.Setsid = true } diff --git a/pkg/command/process_test.go b/pkg/command/process_test.go index da4b185b01..a00fc0024f 100644 --- a/pkg/command/process_test.go +++ b/pkg/command/process_test.go @@ -3,8 +3,12 @@ package command import ( + "os" "os/exec" + "path/filepath" "strconv" + "strings" + "syscall" "testing" "time" ) @@ -44,6 +48,211 @@ func TestKillOnAlreadyExitedProcessIsNoop(t *testing.T) { } } +func TestKillTreeTerminatesGroupAfterWorkerExits(t *testing.T) { + dir := t.TempDir() + childPIDFile := filepath.Join(dir, "child.pid") + // #nosec G204 -- the fixed shell script and all paths are test-controlled. + cmd := exec.Command( + "sh", + "-c", + `sleep 60 >/dev/null 2>&1 & echo $! > "$1"; exit 0`, + "sh", + childPIDFile, + ) + prepareBackgroundTree(cmd) + if err := cmd.Start(); err != nil { + t.Fatalf("start worker: %v", err) + } + workerPID := strconv.Itoa(cmd.Process.Pid) + identity, err := ProcessTreeIdentity(cmd.Process.Pid) + if err != nil { + t.Fatalf("process tree identity: %v", err) + } + t.Cleanup(func() { _ = killTreeAfterWorkerExit(workerPID, "detached-task", identity) }) + + childPID := readTestPID(t, childPIDFile) + + if err := cmd.Wait(); err != nil { + t.Fatalf("wait for worker exit: %v", err) + } + waitForTestProcess(t, childPID, true) + + if err := killTreeAfterWorkerExit(workerPID, "detached-task", identity); err != nil { + t.Fatalf("killTree after worker exit: %v", err) + } + waitForTestProcess(t, childPID, false) +} + +func TestKillTreeAfterWorkerExitWithoutIdentityDoesNotSignalUnrelatedGroup(t *testing.T) { + worker := startUnixTestProcess(t, &syscall.SysProcAttr{Setpgid: true}) + pid := strconv.Itoa(worker.Process.Pid) + if err := killTreeAfterWorkerExit(pid, "stale-task", ""); err == nil { + t.Fatal("killTreeAfterWorkerExit without identity = nil for a live process group") + } + if running, err := isRunning(pid); err != nil || !running { + t.Fatalf("unidentified process running=%t, err=%v", running, err) + } +} + +func TestKillTreeAfterWorkerExitWithMismatchedIdentityDoesNotSignalUnrelatedGroup(t *testing.T) { + worker := startUnixTestProcess(t, &syscall.SysProcAttr{Setpgid: true}) + pid := strconv.Itoa(worker.Process.Pid) + + otherSession := startUnixTestProcess(t, &syscall.SysProcAttr{Setsid: true}) + staleIdentity := processTreeIdentityForTest(t, otherSession.Process.Pid) + workerIdentity := processTreeIdentityForTest(t, worker.Process.Pid) + if staleIdentity == workerIdentity { + t.Fatalf("separate sessions have identical process tree identity %q", workerIdentity) + } + + if err := killTreeAfterWorkerExit(pid, "stale-task", staleIdentity); err != nil { + t.Fatalf("killTreeAfterWorkerExit with stale identity: %v", err) + } + if running, err := isRunning(pid); err != nil || !running { + t.Fatalf("unrelated process running=%t, err=%v", running, err) + } +} + +func TestKillTreeWithMismatchedSavedIdentityDoesNotSignalReusedProcessGroup(t *testing.T) { + worker := startUnixTestProcess(t, &syscall.SysProcAttr{Setpgid: true}) + otherSession := startUnixTestProcess(t, &syscall.SysProcAttr{Setsid: true}) + staleIdentity := processTreeIdentityForTest(t, otherSession.Process.Pid) + workerIdentity := processTreeIdentityForTest(t, worker.Process.Pid) + if staleIdentity == workerIdentity { + t.Fatalf("separate sessions have identical process tree identity %q", workerIdentity) + } + + if err := killTreeWithIdentity( + strconv.Itoa(worker.Process.Pid), + "stale-task", + staleIdentity, + ); err != nil { + t.Fatalf("killTreeWithIdentity with stale identity: %v", err) + } + if running, err := isRunning(strconv.Itoa(worker.Process.Pid)); err != nil || !running { + t.Fatalf("reused process running=%t, err=%v", running, err) + } +} + +func TestKillTreeIdentityRejectsDifferentGroupLeaderStartTime(t *testing.T) { + worker := startUnixTestProcess(t, &syscall.SysProcAttr{Setpgid: true}) + workerIdentity := processTreeIdentityForTest(t, worker.Process.Pid) + identityParts := strings.Split(workerIdentity, ":") + if len(identityParts) < 2 { + t.Fatalf("unexpected process tree identity %q", workerIdentity) + } + startTime, err := strconv.ParseInt(identityParts[len(identityParts)-1], 10, 64) + if err != nil { + t.Fatalf("parse process start identity: %v", err) + } + identityParts[len(identityParts)-1] = strconv.FormatInt(startTime+1, 10) + staleIdentity := strings.Join(identityParts, ":") + + matches, err := processGroupMatchesIdentity(worker.Process.Pid, staleIdentity) + if err != nil { + t.Fatalf("processGroupMatchesIdentity: %v", err) + } + if matches { + t.Fatal("process group matched another process start identity in the same session") + } + if err := killTreeWithIdentity( + strconv.Itoa(worker.Process.Pid), + "stale-task", + staleIdentity, + ); err != nil { + t.Fatalf("killTreeWithIdentity with stale start identity: %v", err) + } + if running, err := isRunning(strconv.Itoa(worker.Process.Pid)); err != nil || !running { + t.Fatalf("worker running=%t, err=%v", running, err) + } +} + +func TestKillTreeWithMissingIdentityDoesNotSignalWorker(t *testing.T) { + worker := startUnixTestProcess(t, &syscall.SysProcAttr{Setpgid: true}) + pid := strconv.Itoa(worker.Process.Pid) + if err := killTreeWithIdentity(pid, "task", ""); err == nil { + t.Fatal("killTreeWithIdentity without saved identity = nil for a live worker") + } + if running, err := isRunning(pid); err != nil || !running { + t.Fatalf("worker running=%t, err=%v", running, err) + } +} + +func TestProcessTreeIdentityMatchesDetachedSession(t *testing.T) { + worker := exec.Command("sleep", "30") + worker.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + if err := worker.Start(); err != nil { + t.Fatalf("start worker: %v", err) + } + defer func() { _ = worker.Process.Kill() }() + + identity, err := ProcessTreeIdentity(worker.Process.Pid) + if err != nil { + t.Fatalf("process tree identity: %v", err) + } + matches, err := processGroupMatchesIdentity(worker.Process.Pid, identity) + if err != nil { + t.Fatalf("match process group identity: %v", err) + } + if !matches { + t.Fatal("process group does not belong to its detached session") + } +} + +func startUnixTestProcess(t *testing.T, attr *syscall.SysProcAttr) *exec.Cmd { + t.Helper() + cmd := exec.Command("sleep", "30") + cmd.SysProcAttr = attr + if err := cmd.Start(); err != nil { + t.Fatalf("start test process: %v", err) + } + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + return cmd +} + +func processTreeIdentityForTest(t *testing.T, pid int) string { + t.Helper() + identity, err := ProcessTreeIdentity(pid) + if err != nil { + t.Fatalf("read process tree identity: %v", err) + } + return identity +} + +func readTestPID(t *testing.T, path string) string { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + // #nosec G304 -- this test reads only its own temporary PID file. + data, err := os.ReadFile(path) + if err == nil { + return strings.TrimSpace(string(data)) + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("worker did not publish PID in %s", path) + return "" +} + +func waitForTestProcess(t *testing.T, pid string, wantRunning bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + running, err := isRunning(pid) + if err != nil { + t.Fatalf("isRunning(%s): %v", pid, err) + } + if running == wantRunning { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("process %s running=%t, want %t", pid, !wantRunning, wantRunning) +} + func TestKillInvalidPIDReturnsError(t *testing.T) { if err := Kill("not-a-pid"); err == nil { t.Error("Kill(\"not-a-pid\") = nil, want error") diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index cd3d88abc2..d983e415fa 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -53,20 +53,24 @@ func killTree(pid, treeName string) error { return err } + var jobErr error + if treeName != "" { + if terminated, err := terminateJob(treeName); err == nil && terminated { + return nil + } else if err != nil { + jobErr = err + } + } + running, err := isRunning(pid) if err != nil { return err } if !running { - return nil - } - - // Job Object termination reaches orphaned descendants; workers launched - // before job ownership existed fall through to taskkill. - if treeName != "" { - if terminated, jobErr := terminateJob(treeName); jobErr == nil && terminated { - return verifyTerminated(parsed) + if jobErr != nil { + return fmt.Errorf("terminate process tree %s: %w", treeName, jobErr) } + return nil } // /T takes down the worker's descendants; /F stands in for SIGKILL, @@ -87,9 +91,78 @@ func killTree(pid, treeName string) error { ) } + if jobErr != nil { + return fmt.Errorf("terminate process tree %s: %w", treeName, jobErr) + } return verifyTerminated(parsed) } +func killTreeWithIdentity(pid, treeName, identity string) error { + parsed, err := parsePID(pid) + if err != nil { + return err + } + if identity == "" { + return fmt.Errorf("process identity is required to terminate worker %d", parsed) + } + if treeName == "" { + return fmt.Errorf("worker job identity is required") + } + currentIdentity, err := processTreeIdentity(parsed) + if err != nil { + return fmt.Errorf("read process identity for worker %d: %w", parsed, err) + } + if currentIdentity == "" { + return fmt.Errorf("process %d exited before its identity was verified", parsed) + } + if currentIdentity != identity { + return fmt.Errorf("process %d identity does not match the saved worker", parsed) + } + terminated, err := terminateJobForPID(treeName, parsed) + if err != nil { + return err + } + if terminated { + return nil + } + running, err := isRunning(pid) + if err != nil { + return err + } + if !running { + return nil + } + return fmt.Errorf("worker job %s is unavailable while process %d is still running", treeName, parsed) +} + +func killTreeAfterWorkerExit(pid, treeName, _ string) error { + if _, err := parsePID(pid); err != nil { + return err + } + if treeName == "" { + return fmt.Errorf("process tree name is required after worker exit") + } + _, err := terminateJob(treeName) + return err +} + +func processTreeIdentity(pid int) (string, error) { + process, err := windows.OpenProcess(windows.PROCESS_QUERY_INFORMATION, false, uint32(pid)) + if err != nil { + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + return "", nil + } + return "", fmt.Errorf("open process %d for identity: %w", pid, err) + } + defer func() { _ = windows.CloseHandle(process) }() + + var created, exited, kernel, user windows.Filetime + if err := windows.GetProcessTimes(process, &created, &exited, &kernel, &user); err != nil { + return "", fmt.Errorf("read process %d creation time: %w", pid, err) + } + return strconv.FormatInt(created.Nanoseconds(), 10), nil +} + // verifyTerminated allows a brief window for the exit to become visible // after the termination call reported success. func verifyTerminated(parsed int) error { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 2b4b39b217..a3023d070f 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -217,6 +217,24 @@ func TestStartDetachedAssignsWorkerToJobBeforeItRuns(t *testing.T) { } } +func TestKillTreeWithIdentityDoesNotFallbackToUnverifiedPID(t *testing.T) { + worker := startHelper(t) + jobName := "devsy-test-no-job-" + strconv.Itoa(worker.Process.Pid) + identity, err := ProcessTreeIdentity(worker.Process.Pid) + if err != nil { + t.Fatalf("ProcessTreeIdentity: %v", err) + } + if identity == "" { + t.Fatal("ProcessTreeIdentity returned empty identity for live worker") + } + if err := killTreeWithIdentity(strconv.Itoa(worker.Process.Pid), jobName, identity); err == nil { + t.Fatal("killTreeWithIdentity without a job object = nil for a live worker") + } + if running, err := isRunning(strconv.Itoa(worker.Process.Pid)); err != nil || !running { + t.Fatalf("worker running=%t, err=%v", running, err) + } +} + func currentProcessInJob() (bool, error) { var inJob int32 result, _, callErr := procIsProcessInJob.Call( @@ -315,6 +333,7 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { helperEnvWaitFile+"="+waitFile, helperEnvPIDFile+"="+childPIDFile, helperEnvChildPIDFile+"="+grandchildPIDFile, + helperEnvExitAfterSpawn+"=1", ) if err := startDetached( parent, @@ -325,7 +344,9 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { t.Fatalf("startDetached: %v", err) } parentPID := waitForPIDFile(t, parentPIDFile) - t.Cleanup(func() { _ = KillTree(strconv.Itoa(parentPID), "devsy-test-worker") }) + t.Cleanup( + func() { _ = KillTreeAfterWorkerExit(strconv.Itoa(parentPID), "devsy-test-worker", "") }, + ) if err := os.WriteFile(waitFile, []byte("go"), 0o600); err != nil { t.Fatalf("signal helper: %v", err) } @@ -333,11 +354,16 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { childPID := waitForPIDFile(t, childPIDFile) grandchildPID := waitForPIDFile(t, grandchildPIDFile) assertNotRunningEventually(t, childPID) - assertRunning(t, parentPID) + assertNotRunningEventually(t, parentPID) + assertNotRunningEventually(t, childPID) assertRunning(t, grandchildPID) - if err := KillTree(strconv.Itoa(parentPID), "devsy-test-worker"); err != nil { - t.Fatalf("Kill(parent): %v", err) + if err := killTreeAfterWorkerExit( + strconv.Itoa(parentPID), + "devsy-test-worker", + "", + ); err != nil { + t.Fatalf("KillTreeAfterWorkerExit(parent): %v", err) } assertNotRunningEventually(t, parentPID) diff --git a/pkg/task/store.go b/pkg/task/store.go index d55595640a..286d1d6b84 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "time" "github.com/devsy-org/devsy/pkg/command" @@ -26,7 +27,8 @@ const pidPublishTimeout = 5 * time.Second type Store struct { dir string // Test seam; see Store.SetKillProcessForTest. - killProcess func(pid, treeName string) error + killProcessWithIdentity func(pid, treeName, identity string) error + killExitedWorkerTree func(pid, treeName, identity string) error // Test seam; see Store.SetAfterClaimForTest. afterClaimForTest func() // Test seam for the cancellation/worker-start lock handoff. @@ -45,7 +47,11 @@ func NewStoreAt(dir string) (*Store, error) { if err := os.MkdirAll(dir, 0o750); err != nil { return nil, fmt.Errorf("create task dir: %w", err) } - return &Store{dir: dir, killProcess: command.KillTree}, nil + return &Store{ + dir: dir, + killProcessWithIdentity: command.KillTreeWithIdentity, + killExitedWorkerTree: command.KillTreeAfterWorkerExit, + }, nil } func (s *Store) Create(opts CreateOptions) (*Task, error) { @@ -147,7 +153,23 @@ func (s *Store) Reconcile(state *State) *State { s.afterClaimForTest() } - return s.failAbandoned(state) + current, err := s.Get(state.ID) + if err != nil { + return state + } + if current.Status.Terminal() { + return current + } + if current.PID != 0 { + if err := s.killExitedWorkerTree( + strconv.Itoa(current.PID), + WorkerProcessName(current.ID), + current.ProcessTreeIdentity, + ); err != nil { + return current + } + } + return s.failAbandoned(current) } // List returns every known task, most recently started first. @@ -209,7 +231,19 @@ func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error // Not in export_test.go: other packages' tests need it, and a _test.go file // compiles only into its own package's test binary. func (s *Store) SetKillProcessForTest(fn func(pid, treeName string) error) { - s.killProcess = fn + s.killProcessWithIdentity = func(pid, treeName, _ string) error { + return fn(pid, treeName) + } + s.killExitedWorkerTree = func(pid, treeName, _ string) error { + return fn(pid, treeName) + } +} + +// SetKillProcessWithIdentityForTest replaces cancellation hooks with an +// identity-aware test implementation. +func (s *Store) SetKillProcessWithIdentityForTest(fn func(pid, treeName, identity string) error) { + s.killProcessWithIdentity = fn + s.killExitedWorkerTree = fn } // SetAfterCancelLockClaimedForTest pauses Cancel while it owns the worker @@ -263,17 +297,6 @@ func (s *Store) failAbandoned(state *State) *State { return reconciled } -// workerAlive guards against signaling a PID whose original worker already -// exited and got recycled to an unrelated process. -func (s *Store) workerAlive(id string) bool { - lock, ok := s.claimDeadWorkerLock(id) - if !ok { - return true - } - _ = lock.Unlock() - return false -} - // tryWorkerLock claims the task's worker lock when no worker owns it. Unlike // claimDeadWorkerLock, it creates the lock file if the worker has not started. func (s *Store) tryWorkerLock(id string) (*flock.Flock, bool, error) { diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index 606a23d91f..c62b9f623a 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -1,6 +1,7 @@ package task import ( + "errors" "testing" "time" ) @@ -99,6 +100,65 @@ func TestActiveForWorkspaceReconcilesAbandonedWorkers(t *testing.T) { } } +func TestActiveForWorkspaceCleansAbandonedProcessTreeBeforeExcluding(t *testing.T) { + store := newTestStore(t) + state := abandonTask(t, store) + if err := store.Open(state.ID).SetWorkspaceID(workspaceOne); err != nil { + t.Fatalf("SetWorkspaceID: %v", err) + } + if err := store.Open(state.ID).SetPID(4242); err != nil { + t.Fatalf("SetPID: %v", err) + } + + killCalls := 0 + store.SetKillProcessForTest(func(pid, treeName string) error { + killCalls++ + if pid != "4242" { + t.Errorf("kill pid = %q, want 4242", pid) + } + if treeName != WorkerProcessName(state.ID) { + t.Errorf("kill tree = %q, want %q", treeName, WorkerProcessName(state.ID)) + } + return nil + }) + + active, err := store.ActiveForWorkspace(workspaceOne, "up") + if err != nil { + t.Fatalf("ActiveForWorkspace: %v", err) + } + if len(active) != 0 { + t.Fatalf("abandoned task remains active after cleanup: %+v", active) + } + if killCalls != 1 { + t.Errorf("process-tree cleanup called %d times, want 1", killCalls) + } +} + +func TestActiveForWorkspaceKeepsAbandonedTaskActiveWhenTreeCleanupFails(t *testing.T) { + store := newTestStore(t) + state := abandonTask(t, store) + if err := store.Open(state.ID).SetWorkspaceID(workspaceOne); err != nil { + t.Fatalf("SetWorkspaceID: %v", err) + } + if err := store.Open(state.ID).SetPID(4242); err != nil { + t.Fatalf("SetPID: %v", err) + } + store.SetKillProcessForTest(func(string, string) error { + return errors.New("descendant cleanup failed") + }) + + active, err := store.ActiveForWorkspace(workspaceOne, "up") + if err != nil { + t.Fatalf("ActiveForWorkspace: %v", err) + } + if len(active) != 1 || active[0].ID != state.ID { + t.Fatalf("task was excluded after failed tree cleanup: %+v", active) + } + if active[0].Status.Terminal() { + t.Fatalf("task marked terminal %q after failed tree cleanup", active[0].Status) + } +} + func TestActiveForWorkspaceTreatsLiveWorkerAsActive(t *testing.T) { store := newTestStore(t) tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) diff --git a/pkg/task/task.go b/pkg/task/task.go index 2a35faf229..ddaf3e4cbf 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -11,6 +11,7 @@ import ( "time" "github.com/devsy-org/devsy/pkg/clierr" + "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/secrets" "github.com/devsy-org/devsy/pkg/status" @@ -45,23 +46,24 @@ func (s Status) Terminal() bool { // doing the work, distinct from whatever process is merely polling this // state. type State struct { - ID string `json:"id"` - Command string `json:"command,omitempty"` - WorkspaceID string `json:"workspaceId,omitempty"` - Status Status `json:"status"` - Phase string `json:"phase,omitempty"` - Step string `json:"step,omitempty"` - OperationID string `json:"operationId,omitempty"` - ParentOperationID string `json:"parentOperationId,omitempty"` - DurationMs int64 `json:"durationMs,omitempty"` - Error string `json:"error,omitempty"` - ErrorCode string `json:"errorCode,omitempty"` - ErrorHint string `json:"errorHint,omitempty"` - ErrorContext map[string]string `json:"errorContext,omitempty"` - Result *config.Result `json:"result,omitempty"` - PID int `json:"pid,omitempty"` - StartedAt time.Time `json:"startedAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + Command string `json:"command,omitempty"` + WorkspaceID string `json:"workspaceId,omitempty"` + Status Status `json:"status"` + Phase string `json:"phase,omitempty"` + Step string `json:"step,omitempty"` + OperationID string `json:"operationId,omitempty"` + ParentOperationID string `json:"parentOperationId,omitempty"` + DurationMs int64 `json:"durationMs,omitempty"` + Error string `json:"error,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorHint string `json:"errorHint,omitempty"` + ErrorContext map[string]string `json:"errorContext,omitempty"` + Result *config.Result `json:"result,omitempty"` + PID int `json:"pid,omitempty"` + ProcessTreeIdentity string `json:"processTreeIdentity,omitempty"` + StartedAt time.Time `json:"startedAt"` + UpdatedAt time.Time `json:"updatedAt"` } // CreateOptions labels a task at creation time for later listing. @@ -83,14 +85,18 @@ type Task struct { func (t *Task) ID() string { return t.id } func (t *Task) SetPID(pid int) error { + identity, err := command.ProcessTreeIdentity(pid) + if err != nil { + return fmt.Errorf("identify process tree for pid %d: %w", pid, err) + } return t.store.update(t.id, func(s *State) { s.PID = pid + s.ProcessTreeIdentity = identity }) } -// HoldWorkerLock claims this task's worker lock for the rest of the process's -// life, marking it as actively being worked on. Callers must not release it: -// the kernel does so when the process exits, crashes, or is killed. +// HoldWorkerLock claims this task's worker lock while its worker is active. +// Release it only when startup aborts before work begins. // // Returns an error if another process already holds the lock, since that means // a worker for this task is already running. @@ -112,12 +118,8 @@ func (t *Task) HoldWorkerLock() error { return nil } -// ReleaseWorkerLockForTest drops the lock HoldWorkerLock acquired, letting a -// test simulate a dead worker. Production code must never call it. -// -// Not in export_test.go: other packages' tests need it, and a _test.go file is -// only compiled into its own package's test binary. -func (t *Task) ReleaseWorkerLockForTest() error { +// ReleaseWorkerLock releases the worker claim when startup aborts before work begins. +func (t *Task) ReleaseWorkerLock() error { if t.workerLock == nil { return nil } @@ -129,6 +131,11 @@ func (t *Task) ReleaseWorkerLockForTest() error { return nil } +// ReleaseWorkerLockForTest simulates a dead worker in tests. +func (t *Task) ReleaseWorkerLockForTest() error { + return t.ReleaseWorkerLock() +} + // SetWorkspaceID corrects the task's workspace label to the resolved ID, // which may differ from whatever label it was created with (e.g. a raw // source string guessed before workspace resolution ran). client.Status @@ -167,6 +174,7 @@ func (t *Task) Succeed(result *config.Result) error { // state. func (t *Task) Cancel() error { var pid int + var identity string observedNonterminal := false err := t.store.update(t.id, func(s *State) { if s.Status.Terminal() { @@ -174,6 +182,7 @@ func (t *Task) Cancel() error { } observedNonterminal = true pid = s.PID + identity = s.ProcessTreeIdentity }) if err != nil { return err @@ -191,17 +200,15 @@ func (t *Task) Cancel() error { if finalized { return nil } + state, err := t.store.Get(t.id) + if err != nil { + return fmt.Errorf("cancel task %s: read process tree identity: %w", t.id, err) + } + identity = state.ProcessTreeIdentity } - if pid != 0 && t.store.workerAlive(t.id) { - if err := t.store.killProcess(strconv.Itoa(pid), WorkerProcessName(t.id)); err != nil { - if !t.store.workerAlive(t.id) { - // The worker exited during the kill attempt, so termination - // is satisfied even though the kill itself reported an error. - return t.store.update(t.id, markCanceled) - } - return fmt.Errorf("cancel task %s: terminate worker pid %d: %w", t.id, pid, err) - } + if pid != 0 { + return t.cancelPublishedPID(pid, identity) } // This invocation observed the task as nonterminal, so a result the worker // committed while unwinding does not win over the accepted cancellation. @@ -249,6 +256,58 @@ func (t *Task) Fail(err error) error { }) } +func (t *Task) cancelPublishedPID(pid int, identity string) error { + workerLock, locked, err := t.store.tryWorkerLock(t.id) + if err != nil { + return fmt.Errorf("cancel task %s: claim worker lock: %w", t.id, err) + } + if locked { + defer func() { _ = workerLock.Unlock() }() + return t.cancelExitedWorkerTree(pid) + } + return t.cancelLiveWorker(pid, identity) +} + +func (t *Task) cancelExitedWorkerTree(pid int) error { + state, err := t.store.Get(t.id) + if err != nil { + return fmt.Errorf("cancel task %s: read process tree identity: %w", t.id, err) + } + if err := t.store.killExitedWorkerTree( + strconv.Itoa(pid), + WorkerProcessName(t.id), + state.ProcessTreeIdentity, + ); err != nil { + return fmt.Errorf("cancel task %s: terminate worker tree: %w", t.id, err) + } + return t.store.update(t.id, markCanceled) +} + +func (t *Task) cancelLiveWorker(pid int, identity string) error { + if err := t.store.killProcessWithIdentity( + strconv.Itoa(pid), + WorkerProcessName(t.id), + identity, + ); err != nil { + workerLock, locked, lockErr := t.store.tryWorkerLock(t.id) + if lockErr != nil { + return fmt.Errorf( + "cancel task %s: terminate worker pid %d: %w (recheck worker lock: %v)", + t.id, + pid, + err, + lockErr, + ) + } + if !locked { + return fmt.Errorf("cancel task %s: terminate worker pid %d: %w", t.id, pid, err) + } + defer func() { _ = workerLock.Unlock() }() + return t.cancelExitedWorkerTree(pid) + } + return t.store.update(t.id, markCanceled) +} + func (t *Task) cancelWithoutPublishedPID() (int, bool, error) { workerLock, locked, err := t.store.tryWorkerLock(t.id) if err != nil { diff --git a/pkg/task/task_pid_supported_test.go b/pkg/task/task_pid_supported_test.go index 7c72c7836b..678ec6f77b 100644 --- a/pkg/task/task_pid_supported_test.go +++ b/pkg/task/task_pid_supported_test.go @@ -4,6 +4,7 @@ package task import ( "os/exec" + "syscall" "testing" "time" ) @@ -17,6 +18,7 @@ func TestCancelSignalsALiveWorkersPID(t *testing.T) { holdWorkerLock(t, tk) worker := exec.Command("sleep", "30") + worker.SysProcAttr = &syscall.SysProcAttr{Setsid: true} if err := worker.Start(); err != nil { t.Fatalf("start: %v", err) } @@ -35,6 +37,54 @@ func TestCancelSignalsALiveWorkersPID(t *testing.T) { } } +func TestCancelUsesPublishedProcessIdentity(t *testing.T) { + store := newTestStore(t) + tk, state := newTaskWithWorkerIdentity(t, store) + if err := tk.HoldWorkerLock(); err != nil { + t.Fatalf("HoldWorkerLock: %v", err) + } + + var gotIdentity string + store.SetKillProcessWithIdentityForTest(func(_, _, identity string) error { + gotIdentity = identity + return nil + }) + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + if state.ProcessTreeIdentity == "" || gotIdentity != state.ProcessTreeIdentity { + t.Fatalf( + "kill identity = %q, want saved identity %q", + gotIdentity, + state.ProcessTreeIdentity, + ) + } +} + +func newTaskWithWorkerIdentity(t *testing.T, store *Store) (*Task, *State) { + t.Helper() + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + worker := exec.Command("sleep", "30") + if err := worker.Start(); err != nil { + t.Fatalf("start worker: %v", err) + } + t.Cleanup(func() { + _ = worker.Process.Kill() + _ = worker.Wait() + }) + if err := tk.SetPID(worker.Process.Pid); err != nil { + t.Fatalf("SetPID: %v", err) + } + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + return tk, state +} + func TestCancelDoesNotSignalAProcessThatReusedThePID(t *testing.T) { store := newTestStore(t) tk, err := store.Create(CreateOptions{}) diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index 28ec155e82..c32f973a3d 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -1040,7 +1040,7 @@ func TestCancelUnpublishedPIDStaysRetryable(t *testing.T) { } } -func TestCancelDeadWorkerDoesNotInvokeKill(t *testing.T) { +func TestCancelDeadWorkerStillCleansUpProcessTree(t *testing.T) { store := newTestStore(t) tk, err := store.Create(CreateOptions{}) if err != nil { @@ -1056,14 +1056,92 @@ func TestCancelDeadWorkerDoesNotInvokeKill(t *testing.T) { t.Fatalf("SetPID: %v", err) } + var actualKill killCall store.SetKillProcessForTest(func(pid, treeName string) error { - t.Error("kill invoked for a task whose worker lock is dead") + actualKill.calls++ + actualKill.pid, actualKill.tree = pid, treeName return nil }) if err := tk.Cancel(); err != nil { t.Fatalf("Cancel: %v", err) } + assertKilledTree(t, actualKill, "4242", WorkerProcessName(tk.ID())) + requireCanceledState(t, store, tk.ID()) +} + +type killCall struct { + calls int + pid string + tree string +} + +func assertKilledTree(t *testing.T, actual killCall, wantPID, wantTree string) { + t.Helper() + if actual.calls != 1 || actual.pid != wantPID || actual.tree != wantTree { + t.Errorf( + "kill calls=(%d, %q, %q), want (1, %q, %q)", + actual.calls, + actual.pid, + actual.tree, + wantPID, + wantTree, + ) + } +} + +func TestCancelDeadWorkerCleanupFailureRemainsRetryable(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := tk.HoldWorkerLock(); err != nil { + t.Fatalf("HoldWorkerLock: %v", err) + } + if err := tk.ReleaseWorkerLockForTest(); err != nil { + t.Fatalf("ReleaseWorkerLockForTest: %v", err) + } + if err := tk.SetPID(4242); err != nil { + t.Fatalf("SetPID: %v", err) + } + store.SetKillProcessForTest(func(string, string) error { + return errors.New("descendant cleanup failed") + }) + + if err := tk.Cancel(); err == nil { + t.Fatal("Cancel = nil, want process-tree cleanup error") + } + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.Status.Terminal() { + t.Fatalf("task recorded terminal %q despite failed tree cleanup", state.Status) + } +} + +func TestCancelUsesIdentityCleanupWhenWorkerExitsDuringKill(t *testing.T) { + store := newTestStore(t) + tk := newLiveWorkerTask(t, store) + killCalls := 0 + store.SetKillProcessForTest(func(string, string) error { + killCalls++ + if killCalls == 1 { + if err := tk.ReleaseWorkerLockForTest(); err != nil { + t.Errorf("ReleaseWorkerLockForTest: %v", err) + } + return errors.New("worker exited before tree lookup") + } + return nil + }) + + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + if killCalls != 2 { + t.Errorf("kill calls = %d, want 2 (live attempt and identity cleanup)", killCalls) + } requireCanceledState(t, store, tk.ID()) } From b523252cf0fe86fb4ba25f1ceb7517921c20edac Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 20:06:14 -0600 Subject: [PATCH 19/49] fix(task): preserve worker job identity across launcher exit --- pkg/command/job_windows.go | 27 ++++++++++++++++- pkg/command/process_windows_test.go | 8 +++-- pkg/task/store_test.go | 4 +-- pkg/task/task.go | 7 +++++ pkg/task/task_test.go | 47 +++++++++++++++++++++-------- pkg/workspace/quiesce_test.go | 11 +++++-- 6 files changed, 82 insertions(+), 22 deletions(-) diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index 84a575b59b..471e9eea0b 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -102,8 +102,21 @@ func ownProcessTree(pid int, workerName string) error { } defer func() { _ = windows.CloseHandle(job) }() + // The worker keeps the name open for cancellation; its exit also closes + // the last handle, which terminates any descendants it leaves behind. + var limits windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION + limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if _, err := windows.SetInformationJobObject( + job, + windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&limits)), + uint32(unsafe.Sizeof(limits)), + ); err != nil { + return fmt.Errorf("configure worker job %s: %w", workerName, err) + } + handle, err := windows.OpenProcess( - windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE|windows.PROCESS_DUP_HANDLE, false, uint32(pid), ) @@ -115,6 +128,18 @@ func ownProcessTree(pid int, workerName string) error { if err := windows.AssignProcessToJobObject(job, handle); err != nil { return fmt.Errorf("assign process %d to job: %w", pid, err) } + var workerJob windows.Handle + if err := windows.DuplicateHandle( + windows.CurrentProcess(), + job, + handle, + &workerJob, + 0, + false, + windows.DUPLICATE_SAME_ACCESS, + ); err != nil { + return fmt.Errorf("keep worker job open in process %d: %w", pid, err) + } return nil } diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index a3023d070f..139c4b4f49 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -200,6 +200,11 @@ func TestStartDetachedAssignsWorkerToJobBeforeItRuns(t *testing.T) { } pid := waitForPIDFile(t, pidFile) t.Cleanup(func() { _ = KillTree(strconv.Itoa(pid), jobName) }) + job, found, err := openWorkerJob(jobName) + if err != nil || !found { + t.Fatalf("open worker job after launcher exit: found=%t, err=%v", found, err) + } + _ = windows.CloseHandle(job) deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { @@ -355,9 +360,6 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { grandchildPID := waitForPIDFile(t, grandchildPIDFile) assertNotRunningEventually(t, childPID) assertNotRunningEventually(t, parentPID) - assertNotRunningEventually(t, childPID) - assertRunning(t, grandchildPID) - if err := killTreeAfterWorkerExit( strconv.Itoa(parentPID), "devsy-test-worker", diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index c62b9f623a..dcda9b2ab0 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -106,7 +106,7 @@ func TestActiveForWorkspaceCleansAbandonedProcessTreeBeforeExcluding(t *testing. if err := store.Open(state.ID).SetWorkspaceID(workspaceOne); err != nil { t.Fatalf("SetWorkspaceID: %v", err) } - if err := store.Open(state.ID).SetPID(4242); err != nil { + if err := setPIDForTest(store.Open(state.ID)); err != nil { t.Fatalf("SetPID: %v", err) } @@ -140,7 +140,7 @@ func TestActiveForWorkspaceKeepsAbandonedTaskActiveWhenTreeCleanupFails(t *testi if err := store.Open(state.ID).SetWorkspaceID(workspaceOne); err != nil { t.Fatalf("SetWorkspaceID: %v", err) } - if err := store.Open(state.ID).SetPID(4242); err != nil { + if err := setPIDForTest(store.Open(state.ID)); err != nil { t.Fatalf("SetPID: %v", err) } store.SetKillProcessForTest(func(string, string) error { diff --git a/pkg/task/task.go b/pkg/task/task.go index ddaf3e4cbf..e2a6d7b738 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -284,6 +284,13 @@ func (t *Task) cancelExitedWorkerTree(pid int) error { } func (t *Task) cancelLiveWorker(pid int, identity string) error { + if identity == "" { + return fmt.Errorf( + "cancel task %s: worker pid %d has no saved process identity; terminate the legacy worker manually and retry", + t.id, + pid, + ) + } if err := t.store.killProcessWithIdentity( strconv.Itoa(pid), WorkerProcessName(t.id), diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index c32f973a3d..30ea7c625f 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -3,6 +3,7 @@ package task import ( "errors" "fmt" + "os" "strings" "sync" "testing" @@ -331,7 +332,7 @@ func TestSetPIDPersists(t *testing.T) { t.Fatalf("Create: %v", err) } - if err := task.SetPID(4242); err != nil { + if err := task.SetPID(os.Getpid()); err != nil { t.Fatalf("SetPID: %v", err) } @@ -339,8 +340,8 @@ func TestSetPIDPersists(t *testing.T) { if err != nil { t.Fatalf("Get: %v", err) } - if state.PID != 4242 { - t.Errorf("PID = %d, want 4242", state.PID) + if state.PID != os.Getpid() { + t.Errorf("PID = %d, want %d", state.PID, os.Getpid()) } } @@ -906,12 +907,19 @@ func newLiveWorkerTask(t *testing.T, store *Store) *Task { t.Fatalf("Create: %v", err) } holdWorkerLock(t, tk) - if err := tk.SetPID(4242); err != nil { - t.Fatalf("SetPID: %v", err) + if err := setPIDForTest(tk); err != nil { + t.Fatalf("setPIDForTest: %v", err) } return tk } +func setPIDForTest(tk *Task) error { + return tk.store.update(tk.id, func(s *State) { + s.PID = 4242 + s.ProcessTreeIdentity = "test-process-tree-identity" + }) +} + func requireCanceledState(t *testing.T, store *Store, id string) { t.Helper() state, err := store.Get(id) @@ -961,6 +969,19 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { requireCanceledState(t, store, tk.ID()) } +func TestCancelLegacyWorkerRequiresManualTermination(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + + err = tk.cancelLiveWorker(4242, "") + if err == nil || !strings.Contains(err.Error(), "terminate the legacy worker manually") { + t.Fatalf("cancelLiveWorker error = %v, want actionable legacy-worker error", err) + } +} + func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { store := newTestStore(t) tk := newLiveWorkerTask(t, store) @@ -998,8 +1019,8 @@ func TestCancelAwaitsPIDPublication(t *testing.T) { go func() { time.Sleep(200 * time.Millisecond) - if err := tk.SetPID(4242); err != nil { - t.Errorf("SetPID: %v", err) + if err := setPIDForTest(tk); err != nil { + t.Errorf("setPIDForTest: %v", err) } }() @@ -1052,8 +1073,8 @@ func TestCancelDeadWorkerStillCleansUpProcessTree(t *testing.T) { if err := tk.ReleaseWorkerLockForTest(); err != nil { t.Fatalf("ReleaseWorkerLockForTest: %v", err) } - if err := tk.SetPID(4242); err != nil { - t.Fatalf("SetPID: %v", err) + if err := setPIDForTest(tk); err != nil { + t.Fatalf("setPIDForTest: %v", err) } var actualKill killCall @@ -1102,8 +1123,8 @@ func TestCancelDeadWorkerCleanupFailureRemainsRetryable(t *testing.T) { if err := tk.ReleaseWorkerLockForTest(); err != nil { t.Fatalf("ReleaseWorkerLockForTest: %v", err) } - if err := tk.SetPID(4242); err != nil { - t.Fatalf("SetPID: %v", err) + if err := setPIDForTest(tk); err != nil { + t.Fatalf("setPIDForTest: %v", err) } store.SetKillProcessForTest(func(string, string) error { return errors.New("descendant cleanup failed") @@ -1152,8 +1173,8 @@ func TestCancelRacingLateSuccessStillWinsAfterSuccessfulTermination(t *testing.T t.Fatalf("Create: %v", err) } holdWorkerLock(t, tk) - if err := tk.SetPID(4242); err != nil { - t.Fatalf("SetPID: %v", err) + if err := setPIDForTest(tk); err != nil { + t.Fatalf("setPIDForTest: %v", err) } store.SetKillProcessForTest(func(pid, treeName string) error { diff --git a/pkg/workspace/quiesce_test.go b/pkg/workspace/quiesce_test.go index a45d838d5a..a2122b033f 100644 --- a/pkg/workspace/quiesce_test.go +++ b/pkg/workspace/quiesce_test.go @@ -2,6 +2,8 @@ package workspace import ( "errors" + "os" + "strconv" "testing" "github.com/devsy-org/devsy/pkg/task" @@ -63,10 +65,13 @@ func TestQuiesceUpTasksAttemptsEveryTaskAndAggregatesFailures(t *testing.T) { tk := tk t.Cleanup(func() { _ = tk.ReleaseWorkerLockForTest() }) } - require.NoError(t, failing.SetPID(1111)) - require.NoError(t, succeeding.SetPID(2222)) + require.NoError(t, failing.SetPID(os.Getpid())) + require.NoError(t, succeeding.SetPID(os.Getpid())) store.SetKillProcessForTest(func(pid, treeName string) error { - if pid == "1111" { + if pid != strconv.Itoa(os.Getpid()) { + t.Errorf("kill pid = %q, want current process %d", pid, os.Getpid()) + } + if treeName == task.WorkerProcessName(failing.ID()) { return errors.New("boom") } return nil From 739f233bd427e01a793aee385bfe1513ee8a2bec Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 20:10:35 -0600 Subject: [PATCH 20/49] refactor(task): simplify Linux process identity checks --- pkg/command/process_identity_linux.go | 111 ++++++++++++++++---------- 1 file changed, 71 insertions(+), 40 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index be801d3db7..ff136d1dad 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -4,6 +4,7 @@ package command import ( "fmt" + "io/fs" "os" "path/filepath" "strconv" @@ -26,17 +27,9 @@ func processTreeIdentity(pid int) (string, error) { } func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { - identityParts := strings.Split(identity, ":") - if len(identityParts) != 2 { - return false, fmt.Errorf("parse process tree identity %q", identity) - } - wantSession, err := strconv.Atoi(identityParts[0]) - if err != nil { - return false, fmt.Errorf("parse session identity %q: %w", identity, err) - } - wantStart, err := strconv.ParseUint(identityParts[1], 10, 64) + wanted, err := parseLinuxProcessTreeIdentity(identity) if err != nil { - return false, fmt.Errorf("parse process start identity %q: %w", identity, err) + return false, err } entries, err := os.ReadDir("/proc") if err != nil { @@ -44,46 +37,84 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { } foundMember := false - // A live group leader must still have the saved start time as well as the session ID. for _, entry := range entries { - pid, err := strconv.Atoi(entry.Name()) + member, leaderMismatch, err := linuxProcessGroupEntryMatches(entry, pgid, wanted) if err != nil { - continue + return false, err } - processDir := filepath.Join("/proc", entry.Name()) - stat, err := os.ReadFile(filepath.Join(processDir, "stat")) - if err != nil { - if !os.IsNotExist(err) { - return false, err - } - continue + if leaderMismatch { + return false, nil } - fields, ok := linuxProcessStatFields(stat) - if !ok || len(fields) < 4 { - continue + foundMember = foundMember || member + } + return foundMember, nil +} + +type linuxProcessTreeIdentity struct { + sessionID int + startTime uint64 +} + +func parseLinuxProcessTreeIdentity(identity string) (linuxProcessTreeIdentity, error) { + identityParts := strings.Split(identity, ":") + if len(identityParts) != 2 { + return linuxProcessTreeIdentity{}, fmt.Errorf("parse process tree identity %q", identity) + } + wanted := linuxProcessTreeIdentity{} + var err error + wanted.sessionID, err = strconv.Atoi(identityParts[0]) + if err != nil { + return linuxProcessTreeIdentity{}, fmt.Errorf("parse session identity %q: %w", identity, err) + } + wanted.startTime, err = strconv.ParseUint(identityParts[1], 10, 64) + if err != nil { + return linuxProcessTreeIdentity{}, fmt.Errorf("parse process start identity %q: %w", identity, err) + } + return wanted, nil +} + +// A live group leader must still have the saved start time as well as the session ID. +func linuxProcessGroupEntryMatches( + entry fs.DirEntry, + pgid int, + wanted linuxProcessTreeIdentity, +) (member, leaderMismatch bool, err error) { + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + return false, false, nil + } + stat, err := os.ReadFile(filepath.Join("/proc", entry.Name(), "stat")) + if err != nil { + if os.IsNotExist(err) { + return false, false, nil } - processGroup, err := strconv.Atoi(fields[2]) - if err != nil || processGroup != pgid { - continue + return false, false, err + } + fields, ok := linuxProcessStatFields(stat) + if !ok || len(fields) < 4 { + return false, false, nil + } + processGroup, err := strconv.Atoi(fields[2]) + if err != nil || processGroup != pgid { + return false, false, nil + } + sessionID, err := strconv.Atoi(fields[3]) + if err != nil { + return false, false, nil + } + if pid == pgid { + if len(fields) < 20 { + return false, false, fmt.Errorf("read process start time for process %d", pid) } - sessionID, err := strconv.Atoi(fields[3]) + leaderStart, err := strconv.ParseUint(fields[19], 10, 64) if err != nil { - continue + return false, false, fmt.Errorf("parse process start time for process %d: %w", pid, err) } - if pid == pgid { - leaderStart, err := strconv.ParseUint(fields[19], 10, 64) - if err != nil { - return false, fmt.Errorf("parse process start time for process %d: %w", pid, err) - } - if sessionID != wantSession || leaderStart != wantStart { - return false, nil - } - } - if sessionID == wantSession { - foundMember = true + if sessionID != wanted.sessionID || leaderStart != wanted.startTime { + return false, true, nil } } - return foundMember, nil + return sessionID == wanted.sessionID, false, nil } func linuxProcessStatFields(stat []byte) ([]string, bool) { From 69d9a7f961298c3a87524ca6b8c824a126b5db0b Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 20:23:25 -0600 Subject: [PATCH 21/49] refactor(task): tighten process checks and test seams --- cmd/workspace/stop.go | 3 +- cmd/workspace/up/detach_test.go | 2 +- .../workspace_client_status_test.go | 4 +- pkg/command/process_supported.go | 14 +++- pkg/command/process_test.go | 9 ++- pkg/task/export_test.go | 7 -- pkg/task/store.go | 47 ++--------- pkg/task/store_test.go | 8 +- pkg/task/task.go | 9 +-- pkg/task/task_pid_supported_test.go | 6 +- pkg/task/task_pid_windows_test.go | 4 +- pkg/task/task_test.go | 81 +++++++++++-------- pkg/workspace/delete.go | 3 +- pkg/workspace/quiesce.go | 12 ++- pkg/workspace/quiesce_test.go | 18 ++--- 15 files changed, 108 insertions(+), 119 deletions(-) delete mode 100644 pkg/task/export_test.go diff --git a/cmd/workspace/stop.go b/cmd/workspace/stop.go index b4668e83e1..7d0287b3dc 100644 --- a/cmd/workspace/stop.go +++ b/cmd/workspace/stop.go @@ -22,8 +22,7 @@ import ( type StopCmd struct { *flags.GlobalFlags client2.StopOptions - // Test seam; see task.Store.SetKillProcessForTest. nil resolves the - // default store. + // Optional quiescence override for focused command tests. quiesceUpTasks func(workspaceID string) error } diff --git a/cmd/workspace/up/detach_test.go b/cmd/workspace/up/detach_test.go index 230ad72f68..71fa484285 100644 --- a/cmd/workspace/up/detach_test.go +++ b/cmd/workspace/up/detach_test.go @@ -108,7 +108,7 @@ func assertWorkerLockReleased(t *testing.T, store *task.Store, id string) { if err := lockHolder.HoldWorkerLock(); err != nil { t.Fatalf("worker lock remained held after canceled startup: %v", err) } - if err := lockHolder.ReleaseWorkerLockForTest(); err != nil { + if err := lockHolder.ReleaseWorkerLock(); err != nil { t.Fatalf("release worker lock: %v", err) } } diff --git a/pkg/client/clientimplementation/workspace_client_status_test.go b/pkg/client/clientimplementation/workspace_client_status_test.go index 1ccf478b17..a93f77e32f 100644 --- a/pkg/client/clientimplementation/workspace_client_status_test.go +++ b/pkg/client/clientimplementation/workspace_client_status_test.go @@ -131,7 +131,7 @@ func TestTaskStatusOverride_AbandonedTaskReportsFailed(t *testing.T) { // Claim then release the worker lock: the kernel frees a dead worker's // lock the same way, which is what marks the task abandoned. require.NoError(t, tk.HoldWorkerLock()) - require.NoError(t, tk.ReleaseWorkerLockForTest()) + require.NoError(t, tk.ReleaseWorkerLock()) s := &workspaceClient{providerWorkspace: &provider.Workspace{ID: testWorkspaceID}} assertOverride(t, s, true, client.StatusFailed) @@ -151,7 +151,7 @@ func TestTaskStatusOverride_LiveWorkerStillReportsProvisioning(t *testing.T) { require.NoError(t, tk.HoldWorkerLock()) // Released on cleanup so the fd closes even if an assertion fails; an open // file can block TempDir removal on some platforms. - t.Cleanup(func() { require.NoError(t, tk.ReleaseWorkerLockForTest()) }) + t.Cleanup(func() { require.NoError(t, tk.ReleaseWorkerLock()) }) s := &workspaceClient{providerWorkspace: &provider.Workspace{ID: testWorkspaceID}} assertOverride(t, s, true, client.StatusProvisioning) diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index d42b3633f1..1ed4cd5a26 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -17,6 +17,9 @@ func isRunning(pid string) (bool, error) { if err != nil { return false, err } + if parsedPid <= 0 { + return false, fmt.Errorf("invalid PID %d: must be positive", parsedPid) + } process, err := os.FindProcess(parsedPid) if err != nil { @@ -25,7 +28,13 @@ func isRunning(pid string) (bool, error) { err = process.Signal(syscall.Signal(0)) if err != nil { - return false, nil + if errors.Is(err, os.ErrProcessDone) || errors.Is(err, syscall.ESRCH) { + return false, nil + } + if errors.Is(err, syscall.EPERM) { + return true, nil + } + return false, fmt.Errorf("check process %d: %w", parsedPid, err) } return true, nil @@ -37,6 +46,9 @@ func killTree(pid, treeName string) error { if err != nil { return err } + if parsedPid <= 0 { + return fmt.Errorf("invalid PID %d: must be positive", parsedPid) + } identity, err := processTreeIdentity(parsedPid) if err != nil { return fmt.Errorf("read process tree identity for worker %d: %w", parsedPid, err) diff --git a/pkg/command/process_test.go b/pkg/command/process_test.go index a00fc0024f..11776c4373 100644 --- a/pkg/command/process_test.go +++ b/pkg/command/process_test.go @@ -254,7 +254,12 @@ func waitForTestProcess(t *testing.T, pid string, wantRunning bool) { } func TestKillInvalidPIDReturnsError(t *testing.T) { - if err := Kill("not-a-pid"); err == nil { - t.Error("Kill(\"not-a-pid\") = nil, want error") + for _, pid := range []string{"not-a-pid", "0", "-5"} { + if _, err := IsRunning(pid); err == nil { + t.Errorf("IsRunning(%q) = nil error, want rejection", pid) + } + if err := Kill(pid); err == nil { + t.Errorf("Kill(%q) = nil, want error", pid) + } } } diff --git a/pkg/task/export_test.go b/pkg/task/export_test.go deleted file mode 100644 index 9ffbc395a5..0000000000 --- a/pkg/task/export_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package task - -// SetAfterClaimForTest runs fn inside Reconcile, just after it claims the dead -// worker's lock. -func (s *Store) SetAfterClaimForTest(fn func()) { - s.afterClaimForTest = fn -} diff --git a/pkg/task/store.go b/pkg/task/store.go index 286d1d6b84..099fb4e37a 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -25,14 +25,11 @@ const pidPublishTimeout = 5 * time.Second // Store persists task state as one JSON file per task under dir. type Store struct { - dir string - // Test seam; see Store.SetKillProcessForTest. - killProcessWithIdentity func(pid, treeName, identity string) error - killExitedWorkerTree func(pid, treeName, identity string) error - // Test seam; see Store.SetAfterClaimForTest. - afterClaimForTest func() - // Test seam for the cancellation/worker-start lock handoff. - afterCancelLockClaimedForTest func() + dir string + killProcessWithIdentity func(pid, treeName, identity string) error + killExitedWorkerTree func(pid, treeName, identity string) error + afterClaimHook func() + afterCancelLockClaimedHook func() } func NewStore() (*Store, error) { @@ -149,8 +146,8 @@ func (s *Store) Reconcile(state *State) *State { } defer func() { _ = lock.Unlock() }() - if s.afterClaimForTest != nil { - s.afterClaimForTest() + if s.afterClaimHook != nil { + s.afterClaimHook() } current, err := s.Get(state.ID) @@ -225,36 +222,6 @@ func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error return active, nil } -// SetKillProcessForTest replaces this store's process termination hook, so a -// test can drive cancellation outcomes deterministically without real PIDs. -// -// Not in export_test.go: other packages' tests need it, and a _test.go file -// compiles only into its own package's test binary. -func (s *Store) SetKillProcessForTest(fn func(pid, treeName string) error) { - s.killProcessWithIdentity = func(pid, treeName, _ string) error { - return fn(pid, treeName) - } - s.killExitedWorkerTree = func(pid, treeName, _ string) error { - return fn(pid, treeName) - } -} - -// SetKillProcessWithIdentityForTest replaces cancellation hooks with an -// identity-aware test implementation. -func (s *Store) SetKillProcessWithIdentityForTest(fn func(pid, treeName, identity string) error) { - s.killProcessWithIdentity = fn - s.killExitedWorkerTree = fn -} - -// SetAfterCancelLockClaimedForTest pauses Cancel while it owns the worker -// lock, allowing tests to verify that a starting worker cannot pass it. -// -// Not in export_test.go: other packages' tests need it, and a _test.go file -// compiles only into its own package's test binary. -func (s *Store) SetAfterCancelLockClaimedForTest(fn func()) { - s.afterCancelLockClaimedForTest = fn -} - // awaitPID polls for the worker to publish its PID, which it does right // after claiming the worker lock. A concurrent cancellation can commit the // terminal state first, leaving nothing to wait for. diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index dcda9b2ab0..f5a176d78b 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -79,8 +79,8 @@ func TestActiveForWorkspaceReconcilesAbandonedWorkers(t *testing.T) { if err := tk.HoldWorkerLock(); err != nil { t.Fatalf("HoldWorkerLock: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } active, err := store.ActiveForWorkspace(workspaceOne, "up") @@ -111,7 +111,7 @@ func TestActiveForWorkspaceCleansAbandonedProcessTreeBeforeExcluding(t *testing. } killCalls := 0 - store.SetKillProcessForTest(func(pid, treeName string) error { + setKillProcessForTest(store, func(pid, treeName string) error { killCalls++ if pid != "4242" { t.Errorf("kill pid = %q, want 4242", pid) @@ -143,7 +143,7 @@ func TestActiveForWorkspaceKeepsAbandonedTaskActiveWhenTreeCleanupFails(t *testi if err := setPIDForTest(store.Open(state.ID)); err != nil { t.Fatalf("SetPID: %v", err) } - store.SetKillProcessForTest(func(string, string) error { + setKillProcessForTest(store, func(string, string) error { return errors.New("descendant cleanup failed") }) diff --git a/pkg/task/task.go b/pkg/task/task.go index e2a6d7b738..fe8f2c5f83 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -131,11 +131,6 @@ func (t *Task) ReleaseWorkerLock() error { return nil } -// ReleaseWorkerLockForTest simulates a dead worker in tests. -func (t *Task) ReleaseWorkerLockForTest() error { - return t.ReleaseWorkerLock() -} - // SetWorkspaceID corrects the task's workspace label to the resolved ID, // which may differ from whatever label it was created with (e.g. a raw // source string guessed before workspace resolution ran). client.Status @@ -322,8 +317,8 @@ func (t *Task) cancelWithoutPublishedPID() (int, bool, error) { } if locked { defer func() { _ = workerLock.Unlock() }() - if t.store.afterCancelLockClaimedForTest != nil { - t.store.afterCancelLockClaimedForTest() + if t.store.afterCancelLockClaimedHook != nil { + t.store.afterCancelLockClaimedHook() } // Holding the worker lock through the state update prevents a // not-yet-started worker from passing its terminal-state check diff --git a/pkg/task/task_pid_supported_test.go b/pkg/task/task_pid_supported_test.go index 678ec6f77b..b46be4378e 100644 --- a/pkg/task/task_pid_supported_test.go +++ b/pkg/task/task_pid_supported_test.go @@ -45,7 +45,7 @@ func TestCancelUsesPublishedProcessIdentity(t *testing.T) { } var gotIdentity string - store.SetKillProcessWithIdentityForTest(func(_, _, identity string) error { + setKillProcessWithIdentityForTest(store, func(_, _, identity string) error { gotIdentity = identity return nil }) @@ -97,8 +97,8 @@ func TestCancelDoesNotSignalAProcessThatReusedThePID(t *testing.T) { if err := tk.HoldWorkerLock(); err != nil { t.Fatalf("HoldWorkerLock: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } innocent := exec.Command("sleep", "30") diff --git a/pkg/task/task_pid_windows_test.go b/pkg/task/task_pid_windows_test.go index 318cf2d7e4..95534d7588 100644 --- a/pkg/task/task_pid_windows_test.go +++ b/pkg/task/task_pid_windows_test.go @@ -53,8 +53,8 @@ func TestCancelDoesNotSignalAProcessThatReusedThePID(t *testing.T) { if err := tk.HoldWorkerLock(); err != nil { t.Fatalf("HoldWorkerLock: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } innocent := sleepWorker() diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index 30ea7c625f..7c5fe4de70 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -384,10 +384,10 @@ func TestCancelHoldsWorkerLockThroughCancellationCommit(t *testing.T) { lockClaimed := make(chan struct{}) continueCancel := make(chan struct{}) - store.SetAfterCancelLockClaimedForTest(func() { + store.afterCancelLockClaimedHook = func() { close(lockClaimed) <-continueCancel - }) + } cancelDone := make(chan error, 1) go func() { cancelDone <- tk.Cancel() }() <-lockClaimed @@ -432,10 +432,10 @@ func TestOverlappingCancelsBeforePIDBothSucceed(t *testing.T) { lockClaimed := make(chan struct{}) continueCancel := make(chan struct{}) - store.SetAfterCancelLockClaimedForTest(func() { + store.afterCancelLockClaimedHook = func() { close(lockClaimed) <-continueCancel - }) + } firstDone := make(chan error, 1) go func() { firstDone <- tk.Cancel() }() <-lockClaimed @@ -646,8 +646,8 @@ func holdWorkerLock(t *testing.T, tk *Task) { t.Fatalf("HoldWorkerLock: %v", err) } t.Cleanup(func() { - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Errorf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Errorf("ReleaseWorkerLock: %v", err) } }) } @@ -707,8 +707,8 @@ func TestAbandonedDetectsReleasedLock(t *testing.T) { if err := tk.HoldWorkerLock(); err != nil { t.Fatalf("HoldWorkerLock: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } state, err := store.Get(tk.ID()) if err != nil { @@ -745,8 +745,8 @@ func abandonTask(t *testing.T, store *Store) *State { if err := tk.HoldWorkerLock(); err != nil { t.Fatalf("HoldWorkerLock: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } state, err := store.Get(tk.ID()) if err != nil { @@ -793,8 +793,8 @@ func TestReconcileDoesNotFailARestartedWorker(t *testing.T) { if err := tk.HoldWorkerLock(); err != nil { t.Fatalf("HoldWorkerLock: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } stale, err := store.Get(tk.ID()) if err != nil { @@ -806,16 +806,16 @@ func TestReconcileDoesNotFailARestartedWorker(t *testing.T) { // Reconcile still holds it. restarted := store.Open(tk.ID()) claimed := false - store.SetAfterClaimForTest(func() { + store.afterClaimHook = func() { claimed = restarted.HoldWorkerLock() == nil - }) + } reconciled := store.Reconcile(stale) if claimed { t.Error("a replacement worker acquired the lock during Reconcile") - if err := restarted.ReleaseWorkerLockForTest(); err != nil { - t.Errorf("ReleaseWorkerLockForTest: %v", err) + if err := restarted.ReleaseWorkerLock(); err != nil { + t.Errorf("ReleaseWorkerLock: %v", err) } } // With the lock held throughout, the reconcile is the only writer and the @@ -842,8 +842,8 @@ func TestReconcileKeepsAWorkerRecordedResult(t *testing.T) { if err := tk.Succeed(nil); err != nil { t.Fatalf("Succeed: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } reconciled := store.Reconcile(stale) @@ -920,6 +920,23 @@ func setPIDForTest(tk *Task) error { }) } +func setKillProcessForTest(store *Store, fn func(pid, treeName string) error) { + store.killProcessWithIdentity = func(pid, treeName, _ string) error { + return fn(pid, treeName) + } + store.killExitedWorkerTree = func(pid, treeName, _ string) error { + return fn(pid, treeName) + } +} + +func setKillProcessWithIdentityForTest( + store *Store, + fn func(pid, treeName, identity string) error, +) { + store.killProcessWithIdentity = fn + store.killExitedWorkerTree = fn +} + func requireCanceledState(t *testing.T, store *Store, id string) { t.Helper() state, err := store.Get(id) @@ -938,7 +955,7 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { tk := newLiveWorkerTask(t, store) killCalls := 0 - store.SetKillProcessForTest(func(pid, treeName string) error { + setKillProcessForTest(store, func(pid, treeName string) error { killCalls++ if killCalls == 1 { return errors.New("boom") @@ -986,7 +1003,7 @@ func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { store := newTestStore(t) tk := newLiveWorkerTask(t, store) - store.SetKillProcessForTest(func(pid, treeName string) error { + setKillProcessForTest(store, func(pid, treeName string) error { state, err := store.Get(tk.ID()) if err != nil { t.Errorf("Get during kill: %v", err) @@ -1012,7 +1029,7 @@ func TestCancelAwaitsPIDPublication(t *testing.T) { holdWorkerLock(t, tk) var killedPID, killedTree string - store.SetKillProcessForTest(func(pid, treeName string) error { + setKillProcessForTest(store, func(pid, treeName string) error { killedPID, killedTree = pid, treeName return nil }) @@ -1044,7 +1061,7 @@ func TestCancelUnpublishedPIDStaysRetryable(t *testing.T) { } holdWorkerLock(t, tk) - store.SetKillProcessForTest(func(pid, treeName string) error { + setKillProcessForTest(store, func(pid, treeName string) error { t.Error("kill invoked before the worker published a pid") return nil }) @@ -1070,15 +1087,15 @@ func TestCancelDeadWorkerStillCleansUpProcessTree(t *testing.T) { if err := tk.HoldWorkerLock(); err != nil { t.Fatalf("HoldWorkerLock: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } if err := setPIDForTest(tk); err != nil { t.Fatalf("setPIDForTest: %v", err) } var actualKill killCall - store.SetKillProcessForTest(func(pid, treeName string) error { + setKillProcessForTest(store, func(pid, treeName string) error { actualKill.calls++ actualKill.pid, actualKill.tree = pid, treeName return nil @@ -1120,13 +1137,13 @@ func TestCancelDeadWorkerCleanupFailureRemainsRetryable(t *testing.T) { if err := tk.HoldWorkerLock(); err != nil { t.Fatalf("HoldWorkerLock: %v", err) } - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Fatalf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) } if err := setPIDForTest(tk); err != nil { t.Fatalf("setPIDForTest: %v", err) } - store.SetKillProcessForTest(func(string, string) error { + setKillProcessForTest(store, func(string, string) error { return errors.New("descendant cleanup failed") }) @@ -1146,11 +1163,11 @@ func TestCancelUsesIdentityCleanupWhenWorkerExitsDuringKill(t *testing.T) { store := newTestStore(t) tk := newLiveWorkerTask(t, store) killCalls := 0 - store.SetKillProcessForTest(func(string, string) error { + setKillProcessForTest(store, func(string, string) error { killCalls++ if killCalls == 1 { - if err := tk.ReleaseWorkerLockForTest(); err != nil { - t.Errorf("ReleaseWorkerLockForTest: %v", err) + if err := tk.ReleaseWorkerLock(); err != nil { + t.Errorf("ReleaseWorkerLock: %v", err) } return errors.New("worker exited before tree lookup") } @@ -1177,7 +1194,7 @@ func TestCancelRacingLateSuccessStillWinsAfterSuccessfulTermination(t *testing.T t.Fatalf("setPIDForTest: %v", err) } - store.SetKillProcessForTest(func(pid, treeName string) error { + setKillProcessForTest(store, func(pid, treeName string) error { // The worker reports its own success while the kill unwinds it. return tk.Succeed(&config.Result{}) }) diff --git a/pkg/workspace/delete.go b/pkg/workspace/delete.go index d3ede1748f..18676c3f8b 100644 --- a/pkg/workspace/delete.go +++ b/pkg/workspace/delete.go @@ -26,8 +26,7 @@ type DeleteOptions struct { Force bool ClientDelete client2.DeleteOptions Owner platform.OwnerFilter - // Test seam; see task.Store.SetKillProcessForTest. nil resolves the - // default store. + // Optional quiescence override for focused deletion tests. quiesceUpTasks func(workspaceID string) error } diff --git a/pkg/workspace/quiesce.go b/pkg/workspace/quiesce.go index a89c73d337..2ce6cefdc3 100644 --- a/pkg/workspace/quiesce.go +++ b/pkg/workspace/quiesce.go @@ -17,6 +17,16 @@ const upTaskCommand = "up" // failures are aggregated, since a surviving worker can reverse the // lifecycle change. func QuiesceUpTasks(store *task.Store, workspaceID string) error { + return quiesceUpTasks(store, workspaceID, func(id string) error { + return store.Open(id).Cancel() + }) +} + +func quiesceUpTasks( + store *task.Store, + workspaceID string, + cancelTask func(id string) error, +) error { active, err := store.ActiveForWorkspace(workspaceID, upTaskCommand) if err != nil { return fmt.Errorf("list active up tasks for workspace %s: %w", workspaceID, err) @@ -24,7 +34,7 @@ func QuiesceUpTasks(store *task.Store, workspaceID string) error { var errs []error for _, state := range active { - if err := store.Open(state.ID).Cancel(); err != nil { + if err := cancelTask(state.ID); err != nil { errs = append(errs, fmt.Errorf("cancel up task %s: %w", state.ID, err)) } } diff --git a/pkg/workspace/quiesce_test.go b/pkg/workspace/quiesce_test.go index a2122b033f..e036e953bf 100644 --- a/pkg/workspace/quiesce_test.go +++ b/pkg/workspace/quiesce_test.go @@ -2,8 +2,6 @@ package workspace import ( "errors" - "os" - "strconv" "testing" "github.com/devsy-org/devsy/pkg/task" @@ -59,25 +57,19 @@ func TestQuiesceUpTasksAttemptsEveryTaskAndAggregatesFailures(t *testing.T) { failing := createUpTask(t, store, "ws-1") succeeding := createUpTask(t, store, "ws-1") - // Both workers read as live; one kill keeps failing. + // Both workers stay active; cancellation of one task fails. for _, tk := range []*task.Task{failing, succeeding} { require.NoError(t, tk.HoldWorkerLock()) tk := tk - t.Cleanup(func() { _ = tk.ReleaseWorkerLockForTest() }) + t.Cleanup(func() { _ = tk.ReleaseWorkerLock() }) } - require.NoError(t, failing.SetPID(os.Getpid())) - require.NoError(t, succeeding.SetPID(os.Getpid())) - store.SetKillProcessForTest(func(pid, treeName string) error { - if pid != strconv.Itoa(os.Getpid()) { - t.Errorf("kill pid = %q, want current process %d", pid, os.Getpid()) - } - if treeName == task.WorkerProcessName(failing.ID()) { + err := quiesceUpTasks(store, "ws-1", func(id string) error { + if id == failing.ID() { return errors.New("boom") } - return nil + return store.Open(id).Fail(task.ErrCanceled) }) - err := QuiesceUpTasks(store, "ws-1") require.Error(t, err) assert.Contains(t, err.Error(), failing.ID()) From f098776cc31556518534ff0db5bad7648b7bea46 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 20:35:54 -0600 Subject: [PATCH 22/49] refactor(task): clarify process identity checks --- pkg/command/job_windows.go | 14 +++++-- pkg/command/process_identity_linux.go | 54 ++++++++++++++++++++------- pkg/command/process_windows.go | 6 ++- pkg/command/process_windows_test.go | 6 ++- 4 files changed, 62 insertions(+), 18 deletions(-) diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index 471e9eea0b..64c7d12dac 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -30,8 +30,12 @@ type jobAccountingInfo struct { totalTerminatedProcesses uint32 } -var procOpenJobObjectW = windows.NewLazySystemDLL("kernel32.dll").NewProc("OpenJobObjectW") -var procIsProcessInJobForTermination = windows.NewLazySystemDLL("kernel32.dll").NewProc("IsProcessInJob") +var ( + procOpenJobObjectW = windows.NewLazySystemDLL("kernel32.dll"). + NewProc("OpenJobObjectW") + procIsProcessInJobForTermination = windows.NewLazySystemDLL("kernel32.dll"). + NewProc("IsProcessInJob") +) // jobNameFor names the Job Object after the worker (e.g. devsy-up-) // rather than its PID: a reused PID must never inherit a predecessor's job. @@ -176,7 +180,11 @@ func terminateJobForPID(workerName string, pid int) (bool, error) { } func processBelongsToJob(pid int, job windows.Handle) (bool, error) { - process, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + process, err := windows.OpenProcess( + windows.PROCESS_QUERY_LIMITED_INFORMATION, + false, + uint32(pid), + ) if err != nil { return false, fmt.Errorf("open worker process %d: %w", pid, err) } diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index ff136d1dad..4e71290fa7 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -64,11 +64,19 @@ func parseLinuxProcessTreeIdentity(identity string) (linuxProcessTreeIdentity, e var err error wanted.sessionID, err = strconv.Atoi(identityParts[0]) if err != nil { - return linuxProcessTreeIdentity{}, fmt.Errorf("parse session identity %q: %w", identity, err) + return linuxProcessTreeIdentity{}, fmt.Errorf( + "parse session identity %q: %w", + identity, + err, + ) } wanted.startTime, err = strconv.ParseUint(identityParts[1], 10, 64) if err != nil { - return linuxProcessTreeIdentity{}, fmt.Errorf("parse process start identity %q: %w", identity, err) + return linuxProcessTreeIdentity{}, fmt.Errorf( + "parse process start identity %q: %w", + identity, + err, + ) } return wanted, nil } @@ -83,14 +91,10 @@ func linuxProcessGroupEntryMatches( if err != nil { return false, false, nil } - stat, err := os.ReadFile(filepath.Join("/proc", entry.Name(), "stat")) + fields, ok, err := linuxProcessStat(pid) if err != nil { - if os.IsNotExist(err) { - return false, false, nil - } return false, false, err } - fields, ok := linuxProcessStatFields(stat) if !ok || len(fields) < 4 { return false, false, nil } @@ -103,20 +107,44 @@ func linuxProcessGroupEntryMatches( return false, false, nil } if pid == pgid { - if len(fields) < 20 { - return false, false, fmt.Errorf("read process start time for process %d", pid) - } - leaderStart, err := strconv.ParseUint(fields[19], 10, 64) + matches, err := linuxProcessLeaderMatches(pid, sessionID, fields, wanted) if err != nil { - return false, false, fmt.Errorf("parse process start time for process %d: %w", pid, err) + return false, false, err } - if sessionID != wanted.sessionID || leaderStart != wanted.startTime { + if !matches { return false, true, nil } } return sessionID == wanted.sessionID, false, nil } +func linuxProcessStat(pid int) ([]string, bool, error) { + stat, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) + if os.IsNotExist(err) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + fields, ok := linuxProcessStatFields(stat) + return fields, ok, nil +} + +func linuxProcessLeaderMatches( + pid, sessionID int, + fields []string, + wanted linuxProcessTreeIdentity, +) (bool, error) { + if len(fields) < 20 { + return false, fmt.Errorf("read process start time for process %d", pid) + } + leaderStart, err := strconv.ParseUint(fields[19], 10, 64) + if err != nil { + return false, fmt.Errorf("parse process start time for process %d: %w", pid, err) + } + return sessionID == wanted.sessionID && leaderStart == wanted.startTime, nil +} + func linuxProcessStatFields(stat []byte) ([]string, bool) { statText := string(stat) closingParen := strings.LastIndexByte(statText, ')') diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index d983e415fa..b9f24154b8 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -132,7 +132,11 @@ func killTreeWithIdentity(pid, treeName, identity string) error { if !running { return nil } - return fmt.Errorf("worker job %s is unavailable while process %d is still running", treeName, parsed) + return fmt.Errorf( + "worker job %s is unavailable while process %d is still running", + treeName, + parsed, + ) } func killTreeAfterWorkerExit(pid, treeName, _ string) error { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 139c4b4f49..1056a388b9 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -232,7 +232,11 @@ func TestKillTreeWithIdentityDoesNotFallbackToUnverifiedPID(t *testing.T) { if identity == "" { t.Fatal("ProcessTreeIdentity returned empty identity for live worker") } - if err := killTreeWithIdentity(strconv.Itoa(worker.Process.Pid), jobName, identity); err == nil { + if err := killTreeWithIdentity( + strconv.Itoa(worker.Process.Pid), + jobName, + identity, + ); err == nil { t.Fatal("killTreeWithIdentity without a job object = nil for a live worker") } if running, err := isRunning(strconv.Itoa(worker.Process.Pid)); err != nil || !running { From c8cf81a8a894a835e51f8c8d0b88b6fe40e064ff Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 20:57:20 -0600 Subject: [PATCH 23/49] fix(task): reject stale process identity handles --- pkg/command/job_windows.go | 22 ++++++++++-- pkg/command/process_identity_linux.go | 18 +++++----- pkg/command/process_identity_linux_test.go | 42 ++++++++++++++++++++++ pkg/command/process_windows_test.go | 18 ++++++++++ 4 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 pkg/command/process_identity_linux_test.go diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index 64c7d12dac..be9bd80f6b 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -31,6 +31,8 @@ type jobAccountingInfo struct { } var ( + procCreateJobObjectForWorker = windows.NewLazySystemDLL("kernel32.dll"). + NewProc("CreateJobObjectW") procOpenJobObjectW = windows.NewLazySystemDLL("kernel32.dll"). NewProc("OpenJobObjectW") procIsProcessInJobForTermination = windows.NewLazySystemDLL("kernel32.dll"). @@ -100,9 +102,9 @@ func ownProcessTree(pid int, workerName string) error { if err != nil { return err } - job, err := windows.CreateJobObject(nil, name) + job, err := createWorkerJobObject(name) if err != nil { - return fmt.Errorf("create job object: %w", err) + return err } defer func() { _ = windows.CloseHandle(job) }() @@ -147,6 +149,22 @@ func ownProcessTree(pid int, workerName string) error { return nil } +func createWorkerJobObject(name *uint16) (windows.Handle, error) { + rawHandle, _, callErr := procCreateJobObjectForWorker.Call( + 0, + uintptr(unsafe.Pointer(name)), + ) + job := windows.Handle(rawHandle) + if job == 0 { + return 0, fmt.Errorf("create job object: %w", callErr) + } + if errors.Is(callErr, windows.ERROR_ALREADY_EXISTS) { + _ = windows.CloseHandle(job) + return 0, fmt.Errorf("job object %q already exists", windows.UTF16PtrToString(name)) + } + return job, nil +} + // terminateJob kills the tree owned by the worker's Job Object. The bool // reports whether a job existed; without one the caller falls back to // taskkill, e.g. for workers launched before job ownership existed. diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index 4e71290fa7..b26791afed 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -3,23 +3,21 @@ package command import ( + "errors" "fmt" "io/fs" "os" "path/filepath" "strconv" "strings" + "syscall" ) func processTreeIdentity(pid int) (string, error) { - stat, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) + fields, ok, err := linuxProcessStat(pid) if err != nil { - if os.IsNotExist(err) { - return "", nil - } return "", err } - fields, ok := linuxProcessStatFields(stat) if !ok || len(fields) < 20 { return "", fmt.Errorf("read session identity for process %d", pid) } @@ -120,16 +118,20 @@ func linuxProcessGroupEntryMatches( func linuxProcessStat(pid int) ([]string, bool, error) { stat, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) - if os.IsNotExist(err) { - return nil, false, nil - } if err != nil { + if linuxProcessStatErrorIsAbsent(err) { + return nil, false, nil + } return nil, false, err } fields, ok := linuxProcessStatFields(stat) return fields, ok, nil } +func linuxProcessStatErrorIsAbsent(err error) bool { + return errors.Is(err, syscall.ESRCH) || os.IsNotExist(err) +} + func linuxProcessLeaderMatches( pid, sessionID int, fields []string, diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go new file mode 100644 index 0000000000..41805ad908 --- /dev/null +++ b/pkg/command/process_identity_linux_test.go @@ -0,0 +1,42 @@ +//go:build linux + +package command + +import ( + "os" + "syscall" + "testing" +) + +func TestLinuxProcessStatErrorIsAbsent(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "process disappeared", + err: &os.PathError{Op: "open", Err: syscall.ESRCH}, + want: true, + }, + { + name: "missing proc entry", + err: &os.PathError{Op: "open", Err: os.ErrNotExist}, + want: true, + }, + {name: "permission denied", err: &os.PathError{Op: "open", Err: os.ErrPermission}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := linuxProcessStatErrorIsAbsent(test.err); got != test.want { + t.Fatalf( + "linuxProcessStatErrorIsAbsent(%v) = %t, want %t", + test.err, + got, + test.want, + ) + } + }) + } +} diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 1056a388b9..f3f2eb6aa5 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -244,6 +244,24 @@ func TestKillTreeWithIdentityDoesNotFallbackToUnverifiedPID(t *testing.T) { } } +func TestOwnProcessTreeRejectsExistingJobObject(t *testing.T) { + workerName := "devsy-test-existing-job-" + strconv.FormatInt(time.Now().UnixNano(), 10) + name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) + if err != nil { + t.Fatalf("convert job name: %v", err) + } + existingJob, err := windows.CreateJobObject(nil, name) + if err != nil { + t.Fatalf("create existing job object: %v", err) + } + defer func() { _ = windows.CloseHandle(existingJob) }() + + err = ownProcessTree(0, workerName) + if err == nil || !strings.Contains(err.Error(), "already exists") { + t.Fatalf("ownProcessTree with an existing job = %v, want already-exists error", err) + } +} + func currentProcessInJob() (bool, error) { var inJob int32 result, _, callErr := procIsProcessInJob.Call( From d9c5216de7118ad55341506fe40e351deef5634c Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 21:09:25 -0600 Subject: [PATCH 24/49] refactor(task): simplify Linux process matching --- pkg/command/process_identity_linux.go | 8 ++++++++ pkg/command/process_identity_linux_test.go | 11 ++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index b26791afed..ebec7fbd86 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -96,6 +96,14 @@ func linuxProcessGroupEntryMatches( if !ok || len(fields) < 4 { return false, false, nil } + return linuxProcessGroupFieldsMatch(pid, pgid, wanted, fields) +} + +func linuxProcessGroupFieldsMatch( + pid, pgid int, + wanted linuxProcessTreeIdentity, + fields []string, +) (member, leaderMismatch bool, err error) { processGroup, err := strconv.Atoi(fields[2]) if err != nil || processGroup != pgid { return false, false, nil diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index 41805ad908..400e46e24e 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -8,6 +8,8 @@ import ( "testing" ) +const processStatOpenOperation = "open" + func TestLinuxProcessStatErrorIsAbsent(t *testing.T) { tests := []struct { name string @@ -16,15 +18,18 @@ func TestLinuxProcessStatErrorIsAbsent(t *testing.T) { }{ { name: "process disappeared", - err: &os.PathError{Op: "open", Err: syscall.ESRCH}, + err: &os.PathError{Op: processStatOpenOperation, Err: syscall.ESRCH}, want: true, }, { name: "missing proc entry", - err: &os.PathError{Op: "open", Err: os.ErrNotExist}, + err: &os.PathError{Op: processStatOpenOperation, Err: os.ErrNotExist}, want: true, }, - {name: "permission denied", err: &os.PathError{Op: "open", Err: os.ErrPermission}}, + { + name: "permission denied", + err: &os.PathError{Op: processStatOpenOperation, Err: os.ErrPermission}, + }, } for _, test := range tests { From 2238b22660df1654cd56bccb69df2524110ef57a Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 26 Sep 2026 22:32:42 -0600 Subject: [PATCH 25/49] fix(task): preserve absent Linux process identity --- pkg/command/process_identity_linux.go | 10 ++++++++-- pkg/command/process_identity_linux_test.go | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index ebec7fbd86..e43f119959 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -18,7 +18,10 @@ func processTreeIdentity(pid int) (string, error) { if err != nil { return "", err } - if !ok || len(fields) < 20 { + if !ok { + return "", nil + } + if len(fields) < 20 { return "", fmt.Errorf("read session identity for process %d", pid) } return fields[3] + ":" + fields[19], nil @@ -133,7 +136,10 @@ func linuxProcessStat(pid int) ([]string, bool, error) { return nil, false, err } fields, ok := linuxProcessStatFields(stat) - return fields, ok, nil + if !ok { + return nil, true, fmt.Errorf("parse process stat for process %d", pid) + } + return fields, true, nil } func linuxProcessStatErrorIsAbsent(err error) bool { diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index 400e46e24e..3626782205 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -45,3 +45,13 @@ func TestLinuxProcessStatErrorIsAbsent(t *testing.T) { }) } } + +func TestProcessTreeIdentityForMissingProcess(t *testing.T) { + identity, err := processTreeIdentity(int(^uint(0) >> 1)) + if err != nil { + t.Fatalf("processTreeIdentity for missing PID: %v", err) + } + if identity != "" { + t.Fatalf("processTreeIdentity for missing PID = %q, want empty identity", identity) + } +} From 0513d46b998aba44602f5792a6d877c206c16e1c Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:12:18 -0600 Subject: [PATCH 26/49] fix(task): address lifecycle review findings --- cmd/workspace/up/detach.go | 27 ++- cmd/workspace/up/detach_test.go | 33 +++- cmd/workspace/up/tunnel.go | 5 +- e2e/tests/ssh/ssh_tunnel_mode.go | 4 +- e2e/tests/ssh/task_cancel.go | 6 +- pkg/command/job_windows.go | 163 ++++++++++++++-- pkg/command/process_identity_linux.go | 32 +++ pkg/command/process_identity_linux_test.go | 32 +++ pkg/command/process_identity_unix.go | 2 +- pkg/command/process_windows.go | 14 +- pkg/command/process_windows_test.go | 217 ++++++++++++++++++++- pkg/task/store.go | 8 +- pkg/task/task.go | 26 +-- pkg/task/task_test.go | 20 +- pkg/workspace/delete.go | 10 +- pkg/workspace/quiesce.go | 7 +- 16 files changed, 527 insertions(+), 79 deletions(-) diff --git a/cmd/workspace/up/detach.go b/cmd/workspace/up/detach.go index 7aa1465e83..fd78d090e7 100644 --- a/cmd/workspace/up/detach.go +++ b/cmd/workspace/up/detach.go @@ -18,13 +18,20 @@ import ( // runDetached submits this invocation as a background task and returns // immediately. func (cmd *UpCmd) runDetached(args []string) error { + if _, err := command.ProcessTreeIdentity(os.Getpid()); err != nil { + return fmt.Errorf("detached up is unsupported on this platform: %w", err) + } store, err := task.NewStore() if err != nil { return err } + workspaceID, err := cmd.detachWorkspaceLabel(args) + if err != nil { + return err + } t, err := store.Create(task.CreateOptions{ Command: "up", - WorkspaceID: cmd.detachWorkspaceLabel(args), + WorkspaceID: workspaceID, }) if err != nil { return err @@ -78,18 +85,22 @@ func detachedArgs(args []string) []string { return out } -// detachWorkspaceLabel resolves the workspace ID at submission so stop and -// delete can match the task before the worker starts and records the -// authoritative ID. -func (cmd *UpCmd) detachWorkspaceLabel(args []string) string { +// detachWorkspaceLabel derives the ID stop and delete use before worker startup. +func (cmd *UpCmd) detachWorkspaceLabel(args []string) (string, error) { + if cmd.FromSnapshot != "" { + if _, err := cmd.resolveExplicitSource(); err != nil { + return "", err + } + } if cmd.ID != "" { - return cmd.ID + return cmd.ID, nil } + args = cmd.ensureArgs(args) if len(args) > 0 { _, source := file.IsLocalDir(args[0]) - return workspace2.ToID(source) + return workspace2.ToID(source), nil } - return "" + return "", nil } func wd() string { diff --git a/cmd/workspace/up/detach_test.go b/cmd/workspace/up/detach_test.go index 71fa484285..4635e1faef 100644 --- a/cmd/workspace/up/detach_test.go +++ b/cmd/workspace/up/detach_test.go @@ -43,7 +43,10 @@ func TestDetachedArgs_NoDetachFlagIsUnchanged(t *testing.T) { func TestDetachWorkspaceLabelNormalizesPath(t *testing.T) { cmd := &UpCmd{} raw := "/tmp/some workspace dir" - got := cmd.detachWorkspaceLabel([]string{raw}) + got, err := cmd.detachWorkspaceLabel([]string{raw}) + if err != nil { + t.Fatalf("detachWorkspaceLabel: %v", err) + } want := workspace2.ToID(raw) if got != want { t.Errorf("detachWorkspaceLabel() = %q, want %q", got, want) @@ -55,7 +58,10 @@ func TestDetachWorkspaceLabelNormalizesPath(t *testing.T) { func TestDetachWorkspaceLabelResolvesCurrentDirectory(t *testing.T) { cmd := &UpCmd{} - got := cmd.detachWorkspaceLabel([]string{"."}) + got, err := cmd.detachWorkspaceLabel([]string{"."}) + if err != nil { + t.Fatalf("detachWorkspaceLabel: %v", err) + } cwd, err := os.Getwd() if err != nil { t.Fatalf("Getwd: %v", err) @@ -66,6 +72,29 @@ func TestDetachWorkspaceLabelResolvesCurrentDirectory(t *testing.T) { } } +func TestDetachWorkspaceLabelUsesSourceFlag(t *testing.T) { + cmd := &UpCmd{} + cmd.Source = "/tmp/source workspace" + got, err := cmd.detachWorkspaceLabel(nil) + if err != nil { + t.Fatalf("detachWorkspaceLabel: %v", err) + } + if want := workspace2.ToID(cmd.Source); got != want { + t.Fatalf("detachWorkspaceLabel() = %q, want %q", got, want) + } +} + +func TestDetachWorkspaceLabelUsesSnapshotWorkspaceID(t *testing.T) { + cmd := &UpCmd{FromSnapshot: testSnapshotRef} + got, err := cmd.detachWorkspaceLabel(nil) + if err != nil { + t.Fatalf("detachWorkspaceLabel: %v", err) + } + if got != "my-ws" { + t.Fatalf("detachWorkspaceLabel() = %q, want my-ws", got) + } +} + func TestOpenTaskExitsWhenAlreadyCanceled(t *testing.T) { // DEVSY_HOME redirects the task store on every platform; $HOME alone // misses the Windows path manager, which reads %LOCALAPPDATA%. diff --git a/cmd/workspace/up/tunnel.go b/cmd/workspace/up/tunnel.go index 213e78e24a..b95a39f7f7 100644 --- a/cmd/workspace/up/tunnel.go +++ b/cmd/workspace/up/tunnel.go @@ -10,9 +10,8 @@ import ( "github.com/devsy-org/devsy/pkg/tunnel" ) -// workspaceTunnelHealth probes whether the workspace behind the tunnel is -// still running. It must stay observational: a health check that dials the -// SSH path can revive a stopped workspace, reversing deliberate stops. +// workspaceTunnelHealth must stay observational; dialing SSH can revive a +// stopped workspace. func workspaceTunnelHealth( client client2.BaseWorkspaceClient, ) func(ctx context.Context) error { diff --git a/e2e/tests/ssh/ssh_tunnel_mode.go b/e2e/tests/ssh/ssh_tunnel_mode.go index fbec59ee04..72232e5f52 100644 --- a/e2e/tests/ssh/ssh_tunnel_mode.go +++ b/e2e/tests/ssh/ssh_tunnel_mode.go @@ -185,9 +185,7 @@ var _ = ginkgo.Describe( return f.DevsySSH(sshCtx, tempDir, "echo iteration-"+strings.Repeat("x", i)) } - // The task phase mixes sub-pipeline events, so the first - // command can still race the tunnel listener on a cold - // container; later commands then exercise the warm path. + // The SSH listener can lag the task's ready phase. var out string gomega.Eventually(func() error { var err error diff --git a/e2e/tests/ssh/task_cancel.go b/e2e/tests/ssh/task_cancel.go index 0ff22875c6..d916641296 100644 --- a/e2e/tests/ssh/task_cancel.go +++ b/e2e/tests/ssh/task_cancel.go @@ -13,9 +13,7 @@ import ( "github.com/onsi/gomega" ) -// stoppedHoldWindow outlasts the tunnel health-check interval and failure -// threshold (30s x 3), so a workspace still Stopped after the window was not -// restarted by the health loop. +// stoppedHoldWindow spans the health loop's 30-second, three-failure threshold. const stoppedHoldWindow = 100 * time.Second var _ = ginkgo.Describe( @@ -54,7 +52,6 @@ var _ = ginkgo.Describe( gomega.Expect(active.PID).NotTo(gomega.BeZero(), "detached up worker should publish its pid") - // On Windows this used to panic with "unsupported". framework.ExpectNoError(f.DevsyWorkspaceTaskCancel(ctx, taskID)) canceled := waitDetachedTaskCanceled(ctx, f, taskID) @@ -94,7 +91,6 @@ var _ = ginkgo.Describe( waitDetachedTunnelReady(ctx, f, taskID) - // Stop discovers the persisted task on its own and cancels it. framework.ExpectNoError(f.DevsyStop(ctx, tempDir)) canceled := waitDetachedTaskCanceled(ctx, f, taskID) diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index be9bd80f6b..dee882cdb1 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -3,13 +3,19 @@ package command import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "os" "os/exec" + "path/filepath" + "strings" "syscall" "time" "unsafe" + "github.com/devsy-org/devsy/pkg/config" "golang.org/x/sys/windows" ) @@ -19,6 +25,8 @@ const jobObjectQueryAccess = 0x0004 const jobTerminationTimeout = 5 * time.Second +const detachedTaskWorkerPrefix = "devsy-up-" + type jobAccountingInfo struct { totalUserTime int64 totalKernelTime int64 @@ -39,10 +47,70 @@ var ( NewProc("IsProcessInJob") ) -// jobNameFor names the Job Object after the worker (e.g. devsy-up-) -// rather than its PID: a reused PID must never inherit a predecessor's job. -func jobNameFor(name string) string { - return name +func jobNameFor(name string) (string, error) { + dataDir := os.Getenv(config.EnvHome) + if dataDir == "" { + localAppData := os.Getenv("LOCALAPPDATA") + if localAppData == "" { + return "", errors.New("resolve Devsy data directory for job object: LOCALAPPDATA is not set") + } + dataDir = filepath.Join(localAppData, config.RepoName) + } + canonicalDir, err := canonicalDirectoryPath(dataDir) + if err != nil { + return "", fmt.Errorf("resolve Devsy data directory for job object: %w", err) + } + dataDirHash := sha256.Sum256([]byte(canonicalDir)) + return "devsy-" + hex.EncodeToString(dataDirHash[:8]) + "-" + name, nil +} + +func canonicalDirectoryPath(path string) (string, error) { + absPath, err := filepath.Abs(path) + if err != nil { + return "", err + } + pathPtr, err := windows.UTF16PtrFromString(absPath) + if err != nil { + return "", err + } + handle, err := windows.CreateFile( + pathPtr, + windows.FILE_READ_ATTRIBUTES, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + windows.OPEN_EXISTING, + windows.FILE_FLAG_BACKUP_SEMANTICS, + 0, + ) + if err != nil { + if errors.Is(err, windows.ERROR_FILE_NOT_FOUND) || errors.Is(err, windows.ERROR_PATH_NOT_FOUND) { + return normalizedWindowsPath(absPath), nil + } + return "", err + } + defer func() { _ = windows.CloseHandle(handle) }() + + const maxPath = 32768 + buffer := make([]uint16, maxPath) + length, err := windows.GetFinalPathNameByHandle(handle, &buffer[0], uint32(len(buffer)), 0) + if err != nil { + return "", err + } + if length >= uint32(len(buffer)) { + return "", fmt.Errorf("canonical directory path exceeds %d UTF-16 code units", len(buffer)) + } + return windows.UTF16ToString(buffer[:length]), nil +} + +func normalizedWindowsPath(path string) string { + path = filepath.Clean(path) + if strings.HasPrefix(path, `\\?\`) { + return path + } + if strings.HasPrefix(path, `\\`) { + return `\\?\UNC\` + strings.TrimPrefix(path, `\\`) + } + return `\\?\` + path } func prepareBackgroundTree(cmd *exec.Cmd) { @@ -52,9 +120,7 @@ func prepareBackgroundTree(cmd *exec.Cmd) { cmd.SysProcAttr.CreationFlags |= windows.CREATE_SUSPENDED } -// resumeBackgroundTree resumes the primary thread of a process created with -// CREATE_SUSPENDED. That process cannot spawn descendants until it has been -// assigned to its Job Object. +// resumeBackgroundTree lets the suspended worker run after job assignment. func resumeBackgroundTree(pid int) error { snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) if err != nil { @@ -93,12 +159,17 @@ func resumeBackgroundTree(pid int) error { } } -// ownProcessTree assigns the process to a Job Object named for the worker. -// Job membership outlives intermediate parents, so tree teardown stays -// complete where taskkill /T loses track. Assignment must succeed before the -// suspended worker is resumed so descendants cannot escape the job. +// ownProcessTree assigns the suspended worker before it can spawn descendants, +// keeping the whole tree in the Job Object. func ownProcessTree(pid int, workerName string) error { - name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) + if _, err := config.DefaultPathManager().DataDir(); err != nil { + return fmt.Errorf("resolve Devsy data directory for job object: %w", err) + } + jobName, err := jobNameFor(workerName) + if err != nil { + return err + } + name, err := windows.UTF16PtrFromString(jobName) if err != nil { return err } @@ -177,13 +248,65 @@ func terminateJob(workerName string) (bool, error) { return terminateJobHandle(job, workerName) } +func terminateJobAfterWorkerExit(workerName string) (bool, error) { + job, found, err := openWorkerJob(workerName) + if err != nil { + return false, err + } + if !found && isDetachedTaskWorkerName(workerName) { + job, found, err = openLegacyWorkerJob(workerName) + if err != nil { + return false, err + } + } + if !found { + return false, nil + } + defer func() { _ = windows.CloseHandle(job) }() + return terminateJobHandle(job, workerName) +} + +func isDetachedTaskWorkerName(workerName string) bool { + if !strings.HasPrefix(workerName, detachedTaskWorkerPrefix) { + return false + } + taskID := strings.TrimPrefix(workerName, detachedTaskWorkerPrefix) + if len(taskID) != 12 { + return false + } + for _, char := range taskID { + if !(char >= 'a' && char <= 'z') && !(char >= 'A' && char <= 'Z') && !(char >= '0' && char <= '9') { + return false + } + } + return true +} + func terminateJobForPID(workerName string, pid int) (bool, error) { job, found, err := openWorkerJob(workerName) + if err != nil { + return false, err + } + if found { + inJob, verifyErr := processBelongsToJob(pid, job) + if verifyErr != nil { + _ = windows.CloseHandle(job) + if errors.Is(verifyErr, windows.ERROR_INVALID_PARAMETER) { + return false, fmt.Errorf("worker process %d exited before job membership was verified", pid) + } + return false, verifyErr + } + if inJob { + defer func() { _ = windows.CloseHandle(job) }() + return terminateJobHandle(job, workerName) + } + _ = windows.CloseHandle(job) + } + job, found, err = openLegacyWorkerJob(workerName) if err != nil || !found { return found, err } defer func() { _ = windows.CloseHandle(job) }() - inJob, err := processBelongsToJob(pid, job) if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { return false, fmt.Errorf("worker process %d exited before job membership was verified", pid) @@ -220,7 +343,19 @@ func processBelongsToJob(pid int, job windows.Handle) (bool, error) { } func openWorkerJob(workerName string) (windows.Handle, bool, error) { - name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) + jobName, err := jobNameFor(workerName) + if err != nil { + return 0, false, err + } + return openNamedJob(jobName, workerName) +} + +func openLegacyWorkerJob(workerName string) (windows.Handle, bool, error) { + return openNamedJob(workerName, workerName) +} + +func openNamedJob(jobName, workerName string) (windows.Handle, bool, error) { + name, err := windows.UTF16PtrFromString(jobName) if err != nil { return 0, false, err } diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index e43f119959..60e234a257 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -32,15 +32,32 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { if err != nil { return false, err } + leaderFields, leaderFound, err := linuxProcessStat(pgid) + if err != nil { + return false, err + } + leaderMatches := false + if leaderFound { + leaderMatches, _, err = linuxProcessGroupFieldsMatch(pgid, pgid, wanted, leaderFields) + if err != nil { + return false, err + } + } entries, err := os.ReadDir("/proc") if err != nil { return false, err } foundMember := false + var skippedStatErr error for _, entry := range entries { member, leaderMismatch, err := linuxProcessGroupEntryMatches(entry, pgid, wanted) if err != nil { + pid, parseErr := strconv.Atoi(entry.Name()) + if parseErr == nil && linuxCanSkipProcessStatError(pid, pgid, leaderMatches, err) { + skippedStatErr = err + continue + } return false, err } if leaderMismatch { @@ -48,6 +65,21 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { } foundMember = foundMember || member } + return linuxProcessGroupMatchResult(pgid, foundMember, skippedStatErr) +} + +func linuxCanSkipProcessStatError(pid, pgid int, leaderMatches bool, err error) bool { + return pid != pgid && leaderMatches && errors.Is(err, fs.ErrPermission) +} + +func linuxProcessGroupMatchResult(pgid int, foundMember bool, skippedStatErr error) (bool, error) { + if skippedStatErr != nil && !foundMember { + return false, fmt.Errorf( + "verify process group %d after unreadable process stats: %w", + pgid, + skippedStatErr, + ) + } return foundMember, nil } diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index 3626782205..c1e5b27ef1 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -46,6 +46,38 @@ func TestLinuxProcessStatErrorIsAbsent(t *testing.T) { } } +func TestLinuxCanSkipUnrelatedUnreadableProcess(t *testing.T) { + permissionErr := &os.PathError{Op: processStatOpenOperation, Err: os.ErrPermission} + tests := []struct { + name string + pid int + pgid int + leaderMatches bool + want bool + }{ + {name: "unrelated process with verified leader", pid: 12, pgid: 7, leaderMatches: true, want: true}, + {name: "worker leader is unreadable", pid: 7, pgid: 7, leaderMatches: true}, + {name: "leader exited", pid: 12, pgid: 7}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := linuxCanSkipProcessStatError(test.pid, test.pgid, test.leaderMatches, permissionErr); got != test.want { + t.Fatalf("linuxCanSkipProcessStatError() = %t, want %t", got, test.want) + } + }) + } +} + +func TestLinuxProcessGroupMatchRequiresAMemberWhenStatsAreUnreadable(t *testing.T) { + permissionErr := &os.PathError{Op: processStatOpenOperation, Err: os.ErrPermission} + if matched, err := linuxProcessGroupMatchResult(7, true, permissionErr); err != nil || !matched { + t.Fatalf("verified group with unreadable unrelated process = (%t, %v), want (true, nil)", matched, err) + } + if matched, err := linuxProcessGroupMatchResult(7, false, permissionErr); err == nil || matched { + t.Fatalf("unverified group with unreadable process = (%t, %v), want error", matched, err) + } +} + func TestProcessTreeIdentityForMissingProcess(t *testing.T) { identity, err := processTreeIdentity(int(^uint(0) >> 1)) if err != nil { diff --git a/pkg/command/process_identity_unix.go b/pkg/command/process_identity_unix.go index 8252bed42b..ef795e2eb1 100644 --- a/pkg/command/process_identity_unix.go +++ b/pkg/command/process_identity_unix.go @@ -5,7 +5,7 @@ package command import "errors" func processTreeIdentity(_ int) (string, error) { - return "", errors.New("process tree identity is unsupported on this platform") + return "", errors.New("process tree identity is implemented on Linux, macOS, and Windows") } func processGroupMatchesIdentity(_ int, _ string) (bool, error) { diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index b9f24154b8..62717e3853 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -55,7 +55,7 @@ func killTree(pid, treeName string) error { var jobErr error if treeName != "" { - if terminated, err := terminateJob(treeName); err == nil && terminated { + if terminated, err := terminateJobForPID(treeName, parsed); err == nil && terminated { return nil } else if err != nil { jobErr = err @@ -139,15 +139,21 @@ func killTreeWithIdentity(pid, treeName, identity string) error { ) } -func killTreeAfterWorkerExit(pid, treeName, _ string) error { +func killTreeAfterWorkerExit(pid, treeName, identity string) error { if _, err := parsePID(pid); err != nil { return err } if treeName == "" { return fmt.Errorf("process tree name is required after worker exit") } - _, err := terminateJob(treeName) - return err + terminated, err := terminateJobAfterWorkerExit(treeName) + if err != nil { + return err + } + if !terminated && identity == "" { + return fmt.Errorf("worker job %s is unavailable for legacy task cleanup", treeName) + } + return nil } func processTreeIdentity(pid int) (string, error) { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index f3f2eb6aa5..982859fefc 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -12,6 +12,7 @@ import ( "time" "unsafe" + "github.com/devsy-org/devsy/pkg/config" "golang.org/x/sys/windows" ) @@ -246,7 +247,11 @@ func TestKillTreeWithIdentityDoesNotFallbackToUnverifiedPID(t *testing.T) { func TestOwnProcessTreeRejectsExistingJobObject(t *testing.T) { workerName := "devsy-test-existing-job-" + strconv.FormatInt(time.Now().UnixNano(), 10) - name, err := windows.UTF16PtrFromString(jobNameFor(workerName)) + jobName, err := jobNameFor(workerName) + if err != nil { + t.Fatalf("resolve job name: %v", err) + } + name, err := windows.UTF16PtrFromString(jobName) if err != nil { t.Fatalf("convert job name: %v", err) } @@ -262,6 +267,216 @@ func TestOwnProcessTreeRejectsExistingJobObject(t *testing.T) { } } +func TestKillTreeWithIdentityFindsLegacyJobObject(t *testing.T) { + worker := startHelper(t) + workerName := "devsy-test-legacy-job-" + strconv.Itoa(worker.Process.Pid) + name, err := windows.UTF16PtrFromString(workerName) + if err != nil { + t.Fatalf("convert legacy job name: %v", err) + } + job, err := windows.CreateJobObject(nil, name) + if err != nil { + t.Fatalf("create legacy job: %v", err) + } + defer func() { _ = windows.CloseHandle(job) }() + + process, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + false, + uint32(worker.Process.Pid), + ) + if err != nil { + t.Fatalf("open helper process: %v", err) + } + defer func() { _ = windows.CloseHandle(process) }() + if err := windows.AssignProcessToJobObject(job, process); err != nil { + t.Fatalf("assign helper to legacy job: %v", err) + } + identity, err := ProcessTreeIdentity(worker.Process.Pid) + if err != nil { + t.Fatalf("ProcessTreeIdentity: %v", err) + } + if err := killTreeWithIdentity(strconv.Itoa(worker.Process.Pid), workerName, identity); err != nil { + t.Fatalf("killTreeWithIdentity: %v", err) + } + assertNotRunningEventually(t, worker.Process.Pid) +} + +func TestKillTreeFindsLegacyJobObject(t *testing.T) { + worker := startHelper(t) + workerName := "devsy-test-legacy-cancel-" + strconv.Itoa(worker.Process.Pid) + name, err := windows.UTF16PtrFromString(workerName) + if err != nil { + t.Fatalf("convert legacy job name: %v", err) + } + job, err := windows.CreateJobObject(nil, name) + if err != nil { + t.Fatalf("create legacy job: %v", err) + } + defer func() { _ = windows.CloseHandle(job) }() + process, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + false, + uint32(worker.Process.Pid), + ) + if err != nil { + t.Fatalf("open helper process: %v", err) + } + defer func() { _ = windows.CloseHandle(process) }() + if err := windows.AssignProcessToJobObject(job, process); err != nil { + t.Fatalf("assign helper to legacy job: %v", err) + } + + if err := KillTree(strconv.Itoa(worker.Process.Pid), workerName); err != nil { + t.Fatalf("KillTree: %v", err) + } + assertNotRunningEventually(t, worker.Process.Pid) +} + +func TestKillTreeAfterWorkerExitFindsLegacyDetachedJob(t *testing.T) { + worker := startHelper(t) + workerName := "devsy-up-abcdefghijkl" + name, err := windows.UTF16PtrFromString(workerName) + if err != nil { + t.Fatalf("convert legacy job name: %v", err) + } + job, err := windows.CreateJobObject(nil, name) + if err != nil { + t.Fatalf("create legacy job: %v", err) + } + defer func() { _ = windows.CloseHandle(job) }() + + process, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + false, + uint32(worker.Process.Pid), + ) + if err != nil { + t.Fatalf("open helper process: %v", err) + } + defer func() { _ = windows.CloseHandle(process) }() + if err := windows.AssignProcessToJobObject(job, process); err != nil { + t.Fatalf("assign helper to legacy job: %v", err) + } + terminated, err := terminateJobAfterWorkerExit(workerName) + if err != nil { + t.Fatalf("terminateJobAfterWorkerExit: %v", err) + } + if !terminated { + t.Fatal("terminateJobAfterWorkerExit found no legacy job") + } + assertNotRunningEventually(t, worker.Process.Pid) +} + +func TestJobNameForSeparatesDevsyHomes(t *testing.T) { + config.ResetPathManager() + t.Cleanup(config.ResetPathManager) + + t.Setenv(config.EnvHome, t.TempDir()) + first, err := jobNameFor("jupyter") + if err != nil { + t.Fatalf("jobNameFor first home: %v", err) + } + + t.Setenv(config.EnvHome, t.TempDir()) + second, err := jobNameFor("jupyter") + if err != nil { + t.Fatalf("jobNameFor second home: %v", err) + } + if first == second { + t.Fatalf("job names collide across Devsy homes: %q", first) + } +} + +func TestJobNameForCanonicalizesEquivalentHomePaths(t *testing.T) { + home := t.TempDir() + t.Setenv(config.EnvHome, home) + first, err := jobNameFor("jupyter") + if err != nil { + t.Fatalf("jobNameFor first path: %v", err) + } + + t.Setenv(config.EnvHome, strings.ToUpper(home)) + second, err := jobNameFor("jupyter") + if err != nil { + t.Fatalf("jobNameFor second path: %v", err) + } + if first != second { + t.Fatalf("equivalent home paths have different job names: %q != %q", first, second) + } +} + +func TestJobNameForDoesNotRequireHomeDirectoryAtCancellation(t *testing.T) { + home := filepath.Join(t.TempDir(), "home") + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatalf("create home: %v", err) + } + t.Setenv(config.EnvHome, home) + first, err := jobNameFor("jupyter") + if err != nil { + t.Fatalf("jobNameFor first call: %v", err) + } + if err := os.Remove(home); err != nil { + t.Fatalf("remove home: %v", err) + } + second, err := jobNameFor("jupyter") + if err != nil { + t.Fatalf("jobNameFor second call: %v", err) + } + if first != second { + t.Fatalf("missing home produced different job names: %q != %q", first, second) + } +} + +func TestTerminateJobForPIDFindsLegacyJobWhenScopedJobIsUnrelated(t *testing.T) { + worker := startHelper(t) + workerName := "devsy-test-legacy-fallback-" + strconv.Itoa(worker.Process.Pid) + scopedName, err := jobNameFor(workerName) + if err != nil { + t.Fatalf("resolve scoped job name: %v", err) + } + scopedNamePtr, err := windows.UTF16PtrFromString(scopedName) + if err != nil { + t.Fatalf("convert scoped job name: %v", err) + } + scopedJob, err := windows.CreateJobObject(nil, scopedNamePtr) + if err != nil { + t.Fatalf("create scoped job: %v", err) + } + defer func() { _ = windows.CloseHandle(scopedJob) }() + + legacyNamePtr, err := windows.UTF16PtrFromString(workerName) + if err != nil { + t.Fatalf("convert legacy job name: %v", err) + } + legacyJob, err := windows.CreateJobObject(nil, legacyNamePtr) + if err != nil { + t.Fatalf("create legacy job: %v", err) + } + defer func() { _ = windows.CloseHandle(legacyJob) }() + process, err := windows.OpenProcess( + windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, + false, + uint32(worker.Process.Pid), + ) + if err != nil { + t.Fatalf("open helper process: %v", err) + } + defer func() { _ = windows.CloseHandle(process) }() + if err := windows.AssignProcessToJobObject(legacyJob, process); err != nil { + t.Fatalf("assign helper to legacy job: %v", err) + } + + terminated, err := terminateJobForPID(workerName, worker.Process.Pid) + if err != nil { + t.Fatalf("terminateJobForPID: %v", err) + } + if !terminated { + t.Fatal("terminateJobForPID found no matching job") + } + assertNotRunningEventually(t, worker.Process.Pid) +} + func currentProcessInJob() (bool, error) { var inJob int32 result, _, callErr := procIsProcessInJob.Call( diff --git a/pkg/task/store.go b/pkg/task/store.go index 099fb4e37a..b00818cdd9 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -28,6 +28,7 @@ type Store struct { dir string killProcessWithIdentity func(pid, treeName, identity string) error killExitedWorkerTree func(pid, treeName, identity string) error + killLegacyWorkerTree func(pid, treeName string) error afterClaimHook func() afterCancelLockClaimedHook func() } @@ -48,6 +49,7 @@ func NewStoreAt(dir string) (*Store, error) { dir: dir, killProcessWithIdentity: command.KillTreeWithIdentity, killExitedWorkerTree: command.KillTreeAfterWorkerExit, + killLegacyWorkerTree: command.KillTree, }, nil } @@ -195,10 +197,8 @@ func (s *Store) List() ([]*State, error) { return states, nil } -// ActiveForWorkspace returns the non-terminal tasks labeled with the exact -// workspace ID, newest first, reconciling stale workers first so an -// abandoned task does not read as live. A non-empty command restricts the -// match. +// ActiveForWorkspace returns non-terminal tasks for a workspace, newest first. +// A non-empty command restricts the match. func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error) { states, err := s.List() if err != nil { diff --git a/pkg/task/task.go b/pkg/task/task.go index fe8f2c5f83..7af3df7387 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -161,12 +161,8 @@ func (t *Task) Succeed(result *config.Result) error { }) } -// Cancel terminates the task's worker before recording the task as canceled: -// the persisted state must never claim the operation is gone while the worker -// performing it is still alive. A kill failure leaves the task nonterminal so -// a later Cancel retries. Cancellation is idempotent: a task already -// terminal, or whose worker already exited, only (re)finalizes the canceled -// state. +// Cancel terminates the worker before recording cancellation. A termination +// failure leaves the task retryable. func (t *Task) Cancel() error { var pid int var identity string @@ -279,18 +275,16 @@ func (t *Task) cancelExitedWorkerTree(pid int) error { } func (t *Task) cancelLiveWorker(pid int, identity string) error { + // The held worker lock ties a legacy PID to its active worker. + pidString := strconv.Itoa(pid) + workerName := WorkerProcessName(t.id) + var err error if identity == "" { - return fmt.Errorf( - "cancel task %s: worker pid %d has no saved process identity; terminate the legacy worker manually and retry", - t.id, - pid, - ) + err = t.store.killLegacyWorkerTree(pidString, workerName) + } else { + err = t.store.killProcessWithIdentity(pidString, workerName, identity) } - if err := t.store.killProcessWithIdentity( - strconv.Itoa(pid), - WorkerProcessName(t.id), - identity, - ); err != nil { + if err != nil { workerLock, locked, lockErr := t.store.tryWorkerLock(t.id) if lockErr != nil { return fmt.Errorf( diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index 7c5fe4de70..0dd1f37485 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -927,6 +927,7 @@ func setKillProcessForTest(store *Store, fn func(pid, treeName string) error) { store.killExitedWorkerTree = func(pid, treeName, _ string) error { return fn(pid, treeName) } + store.killLegacyWorkerTree = fn } func setKillProcessWithIdentityForTest( @@ -986,17 +987,26 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { requireCanceledState(t, store, tk.ID()) } -func TestCancelLegacyWorkerRequiresManualTermination(t *testing.T) { +func TestCancelLegacyLiveWorkerUsesPIDBoundTreeTermination(t *testing.T) { store := newTestStore(t) tk, err := store.Create(CreateOptions{}) if err != nil { t.Fatalf("Create: %v", err) } - - err = tk.cancelLiveWorker(4242, "") - if err == nil || !strings.Contains(err.Error(), "terminate the legacy worker manually") { - t.Fatalf("cancelLiveWorker error = %v, want actionable legacy-worker error", err) + holdWorkerLock(t, tk) + if err := store.update(tk.ID(), func(state *State) { state.PID = 4242 }); err != nil { + t.Fatalf("set legacy PID: %v", err) + } + var actual killCall + store.killLegacyWorkerTree = func(pid, tree string) error { + actual = killCall{calls: actual.calls + 1, pid: pid, tree: tree} + return nil } + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + assertKilledTree(t, actual, "4242", WorkerProcessName(tk.ID())) + requireCanceledState(t, store, tk.ID()) } func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { diff --git a/pkg/workspace/delete.go b/pkg/workspace/delete.go index 18676c3f8b..c50ff72784 100644 --- a/pkg/workspace/delete.go +++ b/pkg/workspace/delete.go @@ -26,7 +26,6 @@ type DeleteOptions struct { Force bool ClientDelete client2.DeleteOptions Owner platform.OwnerFilter - // Optional quiescence override for focused deletion tests. quiesceUpTasks func(workspaceID string) error } @@ -146,13 +145,8 @@ func stopIfRunning( } } -// checkBeforeDelete acquires the lock and verifies the workspace exists; -// force-deletion skips only the status check, never the lock, so the -// quiescence rescan stays serialized with the delete. It returns an unlock -// function that -// must be called by the caller (typically deferred) to release the lock, -// and the resolved workspace status so the caller can decide whether a -// stop is required before delete. +// checkBeforeDelete acquires the workspace lock and returns its status and an +// unlock function. Force skips the status check, not the lock. func checkBeforeDelete( ctx context.Context, client client2.BaseWorkspaceClient, diff --git a/pkg/workspace/quiesce.go b/pkg/workspace/quiesce.go index 2ce6cefdc3..1739de3f1a 100644 --- a/pkg/workspace/quiesce.go +++ b/pkg/workspace/quiesce.go @@ -11,11 +11,8 @@ import ( // upTaskCommand is the command label detached up workers are created with. const upTaskCommand = "up" -// QuiesceUpTasks cancels every persisted, still-active detached up task for -// the workspace so no surviving worker can mutate or restart it during -// stop/delete. Every matching task is attempted even when one fails; -// failures are aggregated, since a surviving worker can reverse the -// lifecycle change. +// QuiesceUpTasks cancels active detached up tasks for the workspace. It tries +// every match and returns all failures because a survivor can restart it. func QuiesceUpTasks(store *task.Store, workspaceID string) error { return quiesceUpTasks(store, workspaceID, func(id string) error { return store.Open(id).Cancel() From 762eb5aa933b7ba5aeba599e36a2aafe6920505e Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:25:00 -0600 Subject: [PATCH 27/49] fix(task): tighten legacy worker cleanup checks --- pkg/command/process_supported.go | 12 ++++++++++-- pkg/command/process_test.go | 6 +++++- pkg/command/process_windows.go | 10 ++++++++-- pkg/command/process_windows_test.go | 30 +++++++++++++++++++++++------ pkg/task/task_pid_windows_test.go | 26 ++++++++++++++++++------- 5 files changed, 66 insertions(+), 18 deletions(-) diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index 1ed4cd5a26..4686fcd9be 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -142,10 +142,18 @@ func killTreeAfterWorkerExit(pid, treeName, identity string) error { } func checkUnidentifiedProcessGroup(pid int, treeName string) error { - if err := syscall.Kill(-pid, 0); errors.Is(err, syscall.ESRCH) { + err := syscall.Kill(-pid, 0) + if errors.Is(err, syscall.ESRCH) { return nil } - return fmt.Errorf("verify process group %d for worker tree %s", pid, treeName) + if err != nil { + return fmt.Errorf("verify process group %d for worker tree %s: %w", pid, treeName, err) + } + return fmt.Errorf( + "verify process group %d for worker tree %s: saved process identity is unavailable", + pid, + treeName, + ) } func signalProcessTree(target int, graceful bool, identity string) error { diff --git a/pkg/command/process_test.go b/pkg/command/process_test.go index 11776c4373..e96fbe5948 100644 --- a/pkg/command/process_test.go +++ b/pkg/command/process_test.go @@ -86,9 +86,13 @@ func TestKillTreeTerminatesGroupAfterWorkerExits(t *testing.T) { func TestKillTreeAfterWorkerExitWithoutIdentityDoesNotSignalUnrelatedGroup(t *testing.T) { worker := startUnixTestProcess(t, &syscall.SysProcAttr{Setpgid: true}) pid := strconv.Itoa(worker.Process.Pid) - if err := killTreeAfterWorkerExit(pid, "stale-task", ""); err == nil { + err := killTreeAfterWorkerExit(pid, "stale-task", "") + if err == nil { t.Fatal("killTreeAfterWorkerExit without identity = nil for a live process group") } + if !strings.Contains(err.Error(), "saved process identity is unavailable") { + t.Fatalf("killTreeAfterWorkerExit error = %v, want missing-identity context", err) + } if running, err := isRunning(pid); err != nil || !running { t.Fatalf("unidentified process running=%t, err=%v", running, err) } diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index 62717e3853..12d5775b95 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -150,8 +150,14 @@ func killTreeAfterWorkerExit(pid, treeName, identity string) error { if err != nil { return err } - if !terminated && identity == "" { - return fmt.Errorf("worker job %s is unavailable for legacy task cleanup", treeName) + if !terminated { + if identity == "" { + return fmt.Errorf( + "worker job %s is unavailable; cannot verify descendants without process identity", + treeName, + ) + } + // SetPID runs only after launch assigns the worker to a kill-on-close job. } return nil } diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 982859fefc..ade8f47947 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -245,6 +245,19 @@ func TestKillTreeWithIdentityDoesNotFallbackToUnverifiedPID(t *testing.T) { } } +func TestKillTreeAfterWorkerExitWithoutIdentityFailsClosedWhenJobIsMissing(t *testing.T) { + worker := startHelper(t) + if err := worker.Process.Kill(); err != nil { + t.Fatalf("kill helper: %v", err) + } + _ = worker.Wait() + + err := killTreeAfterWorkerExit(strconv.Itoa(worker.Process.Pid), "devsy-up-abcdefghijkl", "") + if err == nil || !strings.Contains(err.Error(), "cannot verify descendants without process identity") { + t.Fatalf("killTreeAfterWorkerExit error = %v, want fail-closed identity error", err) + } +} + func TestOwnProcessTreeRejectsExistingJobObject(t *testing.T) { workerName := "devsy-test-existing-job-" + strconv.FormatInt(time.Now().UnixNano(), 10) jobName, err := jobNameFor(workerName) @@ -586,8 +599,15 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { t.Fatalf("startDetached: %v", err) } parentPID := waitForPIDFile(t, parentPIDFile) + identity, err := ProcessTreeIdentity(parentPID) + if err != nil { + t.Fatalf("ProcessTreeIdentity: %v", err) + } + if identity == "" { + t.Fatalf("ProcessTreeIdentity(%d) returned an empty identity", parentPID) + } t.Cleanup( - func() { _ = KillTreeAfterWorkerExit(strconv.Itoa(parentPID), "devsy-test-worker", "") }, + func() { _ = KillTreeAfterWorkerExit(strconv.Itoa(parentPID), "devsy-test-worker", identity) }, ) if err := os.WriteFile(waitFile, []byte("go"), 0o600); err != nil { t.Fatalf("signal helper: %v", err) @@ -597,14 +617,12 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { grandchildPID := waitForPIDFile(t, grandchildPIDFile) assertNotRunningEventually(t, childPID) assertNotRunningEventually(t, parentPID) + assertNotRunningEventually(t, grandchildPID) if err := killTreeAfterWorkerExit( strconv.Itoa(parentPID), "devsy-test-worker", - "", + identity, ); err != nil { - t.Fatalf("KillTreeAfterWorkerExit(parent): %v", err) + t.Fatalf("KillTreeAfterWorkerExit after natural tree cleanup: %v", err) } - - assertNotRunningEventually(t, parentPID) - assertNotRunningEventually(t, grandchildPID) } diff --git a/pkg/task/task_pid_windows_test.go b/pkg/task/task_pid_windows_test.go index 95534d7588..158f53b0ba 100644 --- a/pkg/task/task_pid_windows_test.go +++ b/pkg/task/task_pid_windows_test.go @@ -4,8 +4,12 @@ package task import ( "os/exec" + "strconv" "testing" "time" + + "github.com/devsy-org/devsy/pkg/command" + "github.com/devsy-org/devsy/pkg/config" ) // sleepWorker returns a long-running process using only tools a Windows @@ -15,6 +19,7 @@ func sleepWorker() *exec.Cmd { } func TestCancelSignalsALiveWorkersPID(t *testing.T) { + t.Setenv(config.EnvHome, t.TempDir()) store := newTestStore(t) tk, err := store.Create(CreateOptions{}) if err != nil { @@ -23,11 +28,15 @@ func TestCancelSignalsALiveWorkersPID(t *testing.T) { holdWorkerLock(t, tk) worker := sleepWorker() - if err := worker.Start(); err != nil { - t.Fatalf("start: %v", err) + workerName := WorkerProcessName(tk.ID()) + if err := command.StartBackground(workerName, func() (*exec.Cmd, error) { + return worker, nil + }); err != nil { + t.Fatalf("start background worker: %v", err) } - defer func() { _ = worker.Process.Kill() }() - if err := tk.SetPID(worker.Process.Pid); err != nil { + pid := worker.Process.Pid + t.Cleanup(func() { _ = command.KillTree(strconv.Itoa(pid), workerName) }) + if err := tk.SetPID(pid); err != nil { t.Fatalf("SetPID: %v", err) } @@ -35,9 +44,12 @@ func TestCancelSignalsALiveWorkersPID(t *testing.T) { t.Fatalf("Cancel: %v", err) } - waitErr := worker.Wait() - if waitErr == nil { - t.Fatal("worker process exited cleanly, want it terminated by Cancel") + running, err := command.IsRunning(strconv.Itoa(pid)) + if err != nil { + t.Fatalf("IsRunning: %v", err) + } + if running { + t.Fatalf("worker process %d is still running after Cancel", pid) } } From 0462e8c44297a3dd6aae94dc687b384a02426f77 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:35:43 -0600 Subject: [PATCH 28/49] fix(command): simplify Linux process group verification --- pkg/command/process_identity_linux.go | 45 +++++++++++++++++----- pkg/command/process_identity_linux_test.go | 8 +++- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index 60e234a257..3da7aca05e 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -47,25 +47,52 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { if err != nil { return false, err } + foundMember, leaderMismatch, skippedStatErr, err := linuxProcessGroupEntriesMatch( + entries, + pgid, + wanted, + leaderMatches, + ) + if err != nil { + return false, err + } + if leaderMismatch { + return false, nil + } + return linuxProcessGroupMatchResult(pgid, foundMember, skippedStatErr) +} - foundMember := false - var skippedStatErr error +func linuxProcessGroupEntriesMatch( + entries []os.DirEntry, + pgid int, + wanted linuxProcessTreeIdentity, + leaderMatches bool, +) (foundMember, leaderMismatch bool, skippedStatErr, scanErr error) { for _, entry := range entries { - member, leaderMismatch, err := linuxProcessGroupEntryMatches(entry, pgid, wanted) + member, mismatchedLeader, err := linuxProcessGroupEntryMatches(entry, pgid, wanted) if err != nil { - pid, parseErr := strconv.Atoi(entry.Name()) - if parseErr == nil && linuxCanSkipProcessStatError(pid, pgid, leaderMatches, err) { + if linuxProcessGroupEntryErrorCanBeSkipped(entry, pgid, leaderMatches, err) { skippedStatErr = err continue } - return false, err + return false, false, skippedStatErr, err } - if leaderMismatch { - return false, nil + if mismatchedLeader { + return false, true, skippedStatErr, nil } foundMember = foundMember || member } - return linuxProcessGroupMatchResult(pgid, foundMember, skippedStatErr) + return foundMember, false, skippedStatErr, nil +} + +func linuxProcessGroupEntryErrorCanBeSkipped( + entry os.DirEntry, + pgid int, + leaderMatches bool, + err error, +) bool { + pid, parseErr := strconv.Atoi(entry.Name()) + return parseErr == nil && linuxCanSkipProcessStatError(pid, pgid, leaderMatches, err) } func linuxCanSkipProcessStatError(pid, pgid int, leaderMatches bool, err error) bool { diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index c1e5b27ef1..9df3d14263 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -55,7 +55,13 @@ func TestLinuxCanSkipUnrelatedUnreadableProcess(t *testing.T) { leaderMatches bool want bool }{ - {name: "unrelated process with verified leader", pid: 12, pgid: 7, leaderMatches: true, want: true}, + { + name: "unrelated process with verified leader", + pid: 12, + pgid: 7, + leaderMatches: true, + want: true, + }, {name: "worker leader is unreadable", pid: 7, pgid: 7, leaderMatches: true}, {name: "leader exited", pid: 12, pgid: 7}, } From 883f9cfde106df6e8344cb12395fd8fe8cefe122 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:42:30 -0600 Subject: [PATCH 29/49] fix(command): satisfy Linux process group lint rules --- pkg/command/process_identity_linux.go | 26 ++++++++++++++-------- pkg/command/process_identity_linux_test.go | 8 ++++++- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index 3da7aca05e..b6ef08530a 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -47,7 +47,7 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { if err != nil { return false, err } - foundMember, leaderMismatch, skippedStatErr, err := linuxProcessGroupEntriesMatch( + scan, err := linuxProcessGroupEntriesMatch( entries, pgid, wanted, @@ -56,10 +56,16 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { if err != nil { return false, err } - if leaderMismatch { + if scan.leaderMismatch { return false, nil } - return linuxProcessGroupMatchResult(pgid, foundMember, skippedStatErr) + return linuxProcessGroupMatchResult(pgid, scan.foundMember, scan.skippedStatErr) +} + +type linuxProcessGroupScan struct { + foundMember bool + leaderMismatch bool + skippedStatErr error } func linuxProcessGroupEntriesMatch( @@ -67,22 +73,24 @@ func linuxProcessGroupEntriesMatch( pgid int, wanted linuxProcessTreeIdentity, leaderMatches bool, -) (foundMember, leaderMismatch bool, skippedStatErr, scanErr error) { +) (linuxProcessGroupScan, error) { + var scan linuxProcessGroupScan for _, entry := range entries { member, mismatchedLeader, err := linuxProcessGroupEntryMatches(entry, pgid, wanted) if err != nil { if linuxProcessGroupEntryErrorCanBeSkipped(entry, pgid, leaderMatches, err) { - skippedStatErr = err + scan.skippedStatErr = err continue } - return false, false, skippedStatErr, err + return scan, err } if mismatchedLeader { - return false, true, skippedStatErr, nil + scan.leaderMismatch = true + return scan, nil } - foundMember = foundMember || member + scan.foundMember = scan.foundMember || member } - return foundMember, false, skippedStatErr, nil + return scan, nil } func linuxProcessGroupEntryErrorCanBeSkipped( diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index 9df3d14263..ad010ebf75 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -67,7 +67,13 @@ func TestLinuxCanSkipUnrelatedUnreadableProcess(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if got := linuxCanSkipProcessStatError(test.pid, test.pgid, test.leaderMatches, permissionErr); got != test.want { + got := linuxCanSkipProcessStatError( + test.pid, + test.pgid, + test.leaderMatches, + permissionErr, + ) + if got != test.want { t.Fatalf("linuxCanSkipProcessStatError() = %t, want %t", got, test.want) } }) From 8b3df5ff0b6df8c6ab64c28bb9ee54a9fa0ca68d Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:46:59 -0600 Subject: [PATCH 30/49] style(command): satisfy golines in Linux tests --- pkg/command/process_identity_linux_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index ad010ebf75..7d8c12d493 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -82,10 +82,16 @@ func TestLinuxCanSkipUnrelatedUnreadableProcess(t *testing.T) { func TestLinuxProcessGroupMatchRequiresAMemberWhenStatsAreUnreadable(t *testing.T) { permissionErr := &os.PathError{Op: processStatOpenOperation, Err: os.ErrPermission} - if matched, err := linuxProcessGroupMatchResult(7, true, permissionErr); err != nil || !matched { - t.Fatalf("verified group with unreadable unrelated process = (%t, %v), want (true, nil)", matched, err) + matched, err := linuxProcessGroupMatchResult(7, true, permissionErr) + if err != nil || !matched { + t.Fatalf( + "verified group with unreadable unrelated process = (%t, %v), want (true, nil)", + matched, + err, + ) } - if matched, err := linuxProcessGroupMatchResult(7, false, permissionErr); err == nil || matched { + matched, err = linuxProcessGroupMatchResult(7, false, permissionErr) + if err == nil || matched { t.Fatalf("unverified group with unreadable process = (%t, %v), want error", matched, err) } } From 0a2bb1813e51d2325f1538aec5770adb98445d4e Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:59:08 -0600 Subject: [PATCH 31/49] style(command): apply golangci-lint formatting --- pkg/command/job_windows.go | 15 +++++++++++---- pkg/command/process_windows_test.go | 9 +++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index dee882cdb1..1de7d8aa87 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -52,7 +52,9 @@ func jobNameFor(name string) (string, error) { if dataDir == "" { localAppData := os.Getenv("LOCALAPPDATA") if localAppData == "" { - return "", errors.New("resolve Devsy data directory for job object: LOCALAPPDATA is not set") + return "", errors.New( + "resolve Devsy data directory for job object: LOCALAPPDATA is not set", + ) } dataDir = filepath.Join(localAppData, config.RepoName) } @@ -83,7 +85,8 @@ func canonicalDirectoryPath(path string) (string, error) { 0, ) if err != nil { - if errors.Is(err, windows.ERROR_FILE_NOT_FOUND) || errors.Is(err, windows.ERROR_PATH_NOT_FOUND) { + if errors.Is(err, windows.ERROR_FILE_NOT_FOUND) || + errors.Is(err, windows.ERROR_PATH_NOT_FOUND) { return normalizedWindowsPath(absPath), nil } return "", err @@ -275,7 +278,8 @@ func isDetachedTaskWorkerName(workerName string) bool { return false } for _, char := range taskID { - if !(char >= 'a' && char <= 'z') && !(char >= 'A' && char <= 'Z') && !(char >= '0' && char <= '9') { + if !(char >= 'a' && char <= 'z') && !(char >= 'A' && char <= 'Z') && + !(char >= '0' && char <= '9') { return false } } @@ -292,7 +296,10 @@ func terminateJobForPID(workerName string, pid int) (bool, error) { if verifyErr != nil { _ = windows.CloseHandle(job) if errors.Is(verifyErr, windows.ERROR_INVALID_PARAMETER) { - return false, fmt.Errorf("worker process %d exited before job membership was verified", pid) + return false, fmt.Errorf( + "worker process %d exited before job membership was verified", + pid, + ) } return false, verifyErr } diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index ade8f47947..cb9423b5cf 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -253,7 +253,8 @@ func TestKillTreeAfterWorkerExitWithoutIdentityFailsClosedWhenJobIsMissing(t *te _ = worker.Wait() err := killTreeAfterWorkerExit(strconv.Itoa(worker.Process.Pid), "devsy-up-abcdefghijkl", "") - if err == nil || !strings.Contains(err.Error(), "cannot verify descendants without process identity") { + if err == nil || + !strings.Contains(err.Error(), "cannot verify descendants without process identity") { t.Fatalf("killTreeAfterWorkerExit error = %v, want fail-closed identity error", err) } } @@ -309,7 +310,11 @@ func TestKillTreeWithIdentityFindsLegacyJobObject(t *testing.T) { if err != nil { t.Fatalf("ProcessTreeIdentity: %v", err) } - if err := killTreeWithIdentity(strconv.Itoa(worker.Process.Pid), workerName, identity); err != nil { + if err := killTreeWithIdentity( + strconv.Itoa(worker.Process.Pid), + workerName, + identity, + ); err != nil { t.Fatalf("killTreeWithIdentity: %v", err) } assertNotRunningEventually(t, worker.Process.Pid) From a0fb202b3852591c8184138fd7edcc723a615027 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 02:04:39 -0600 Subject: [PATCH 32/49] style(task): remove redundant comments --- cmd/workspace/up/detach.go | 1 - pkg/task/store_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/cmd/workspace/up/detach.go b/cmd/workspace/up/detach.go index fd78d090e7..e6aa154810 100644 --- a/cmd/workspace/up/detach.go +++ b/cmd/workspace/up/detach.go @@ -121,7 +121,6 @@ func (cmd *UpCmd) openTask() (*task.Task, error) { return nil, err } t := store.Open(cmd.taskID) - // obtain the worker lock first. if err := t.HoldWorkerLock(); err != nil { failTask(t, err) return nil, err diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index f5a176d78b..7688ab4968 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -37,7 +37,6 @@ func TestActiveForWorkspaceFilters(t *testing.T) { t.Fatalf("Succeed: %v", err) } - // The command filter keeps only live up tasks for the workspace. active := queryActive(t, store, workspaceOne, "up") if len(active) != 1 || active[0].ID != up.ID() { t.Errorf("command-filtered match = %+v, want only the live up task", active) From d6c8362b27c87978e48723d5a2e380aee1e27118 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 02:37:35 -0600 Subject: [PATCH 33/49] fix(command): continue Linux process group scans --- pkg/command/process_identity_linux.go | 25 +++++---- pkg/command/process_identity_linux_test.go | 61 ++++++++++++++++------ 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index b6ef08530a..7661b97979 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -36,12 +36,19 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { if err != nil { return false, err } - leaderMatches := false if leaderFound { - leaderMatches, _, err = linuxProcessGroupFieldsMatch(pgid, pgid, wanted, leaderFields) + _, leaderMismatch, err := linuxProcessGroupFieldsMatch( + pgid, + pgid, + wanted, + leaderFields, + ) if err != nil { return false, err } + if leaderMismatch { + return false, nil + } } entries, err := os.ReadDir("/proc") if err != nil { @@ -51,7 +58,6 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { entries, pgid, wanted, - leaderMatches, ) if err != nil { return false, err @@ -72,13 +78,12 @@ func linuxProcessGroupEntriesMatch( entries []os.DirEntry, pgid int, wanted linuxProcessTreeIdentity, - leaderMatches bool, ) (linuxProcessGroupScan, error) { var scan linuxProcessGroupScan for _, entry := range entries { member, mismatchedLeader, err := linuxProcessGroupEntryMatches(entry, pgid, wanted) if err != nil { - if linuxProcessGroupEntryErrorCanBeSkipped(entry, pgid, leaderMatches, err) { + if linuxProcessGroupEntryErrorCanBeSkipped(entry, pgid, err) { scan.skippedStatErr = err continue } @@ -96,19 +101,21 @@ func linuxProcessGroupEntriesMatch( func linuxProcessGroupEntryErrorCanBeSkipped( entry os.DirEntry, pgid int, - leaderMatches bool, err error, ) bool { pid, parseErr := strconv.Atoi(entry.Name()) - return parseErr == nil && linuxCanSkipProcessStatError(pid, pgid, leaderMatches, err) + return parseErr == nil && linuxCanSkipProcessStatError(pid, pgid, err) } -func linuxCanSkipProcessStatError(pid, pgid int, leaderMatches bool, err error) bool { - return pid != pgid && leaderMatches && errors.Is(err, fs.ErrPermission) +func linuxCanSkipProcessStatError(pid, pgid int, err error) bool { + return pid != pgid && errors.Is(err, fs.ErrPermission) } func linuxProcessGroupMatchResult(pgid int, foundMember bool, skippedStatErr error) (bool, error) { if skippedStatErr != nil && !foundMember { + if errors.Is(syscall.Kill(-pgid, 0), syscall.ESRCH) { + return false, nil + } return false, fmt.Errorf( "verify process group %d after unreadable process stats: %w", pgid, diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index 7d8c12d493..c9a813c5f2 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -4,6 +4,7 @@ package command import ( "os" + "os/exec" "syscall" "testing" ) @@ -49,28 +50,25 @@ func TestLinuxProcessStatErrorIsAbsent(t *testing.T) { func TestLinuxCanSkipUnrelatedUnreadableProcess(t *testing.T) { permissionErr := &os.PathError{Op: processStatOpenOperation, Err: os.ErrPermission} tests := []struct { - name string - pid int - pgid int - leaderMatches bool - want bool + name string + pid int + pgid int + want bool }{ { - name: "unrelated process with verified leader", - pid: 12, - pgid: 7, - leaderMatches: true, - want: true, + name: "unrelated process with verified leader", + pid: 12, + pgid: 7, + want: true, }, - {name: "worker leader is unreadable", pid: 7, pgid: 7, leaderMatches: true}, - {name: "leader exited", pid: 12, pgid: 7}, + {name: "worker leader is unreadable", pid: 7, pgid: 7}, + {name: "leader exited", pid: 12, pgid: 7, want: true}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got := linuxCanSkipProcessStatError( test.pid, test.pgid, - test.leaderMatches, permissionErr, ) if got != test.want { @@ -82,7 +80,8 @@ func TestLinuxCanSkipUnrelatedUnreadableProcess(t *testing.T) { func TestLinuxProcessGroupMatchRequiresAMemberWhenStatsAreUnreadable(t *testing.T) { permissionErr := &os.PathError{Op: processStatOpenOperation, Err: os.ErrPermission} - matched, err := linuxProcessGroupMatchResult(7, true, permissionErr) + pgid := startLinuxProcessGroupHelper(t) + matched, err := linuxProcessGroupMatchResult(pgid, true, permissionErr) if err != nil || !matched { t.Fatalf( "verified group with unreadable unrelated process = (%t, %v), want (true, nil)", @@ -90,12 +89,44 @@ func TestLinuxProcessGroupMatchRequiresAMemberWhenStatsAreUnreadable(t *testing. err, ) } - matched, err = linuxProcessGroupMatchResult(7, false, permissionErr) + matched, err = linuxProcessGroupMatchResult(pgid, false, permissionErr) if err == nil || matched { t.Fatalf("unverified group with unreadable process = (%t, %v), want error", matched, err) } } +func TestLinuxProcessGroupMatchTreatsMissingGroupAsGone(t *testing.T) { + permissionErr := &os.PathError{Op: processStatOpenOperation, Err: os.ErrPermission} + const largestPID = int(^uint32(0) >> 1) + + matched, err := linuxProcessGroupMatchResult(largestPID, false, permissionErr) + if err != nil || matched { + t.Fatalf("missing process group = (%t, %v), want (false, nil)", matched, err) + } +} + +func TestLinuxProcessGroupProbeHelper(t *testing.T) { + if os.Getenv("DEVSY_PROCESS_GROUP_HELPER") != "1" { + return + } + select {} +} + +func startLinuxProcessGroupHelper(t *testing.T) int { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestLinuxProcessGroupProbeHelper$") + cmd.Env = append(os.Environ(), "DEVSY_PROCESS_GROUP_HELPER=1") + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + if err := cmd.Start(); err != nil { + t.Fatalf("start process group helper: %v", err) + } + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + return cmd.Process.Pid +} + func TestProcessTreeIdentityForMissingProcess(t *testing.T) { identity, err := processTreeIdentity(int(^uint(0) >> 1)) if err != nil { From 9e8f4db4bcf70c5d61a1b79e4e938cb8726c715a Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:07:54 -0600 Subject: [PATCH 34/49] fix(command): satisfy process identity lint --- pkg/command/process_identity_linux.go | 29 +++++++++++----------- pkg/command/process_identity_linux_test.go | 1 + 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index 7661b97979..127ab57bfe 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -32,23 +32,12 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { if err != nil { return false, err } - leaderFields, leaderFound, err := linuxProcessStat(pgid) + leaderMismatch, err := linuxProcessGroupLeaderMismatch(pgid, wanted) if err != nil { return false, err } - if leaderFound { - _, leaderMismatch, err := linuxProcessGroupFieldsMatch( - pgid, - pgid, - wanted, - leaderFields, - ) - if err != nil { - return false, err - } - if leaderMismatch { - return false, nil - } + if leaderMismatch { + return false, nil } entries, err := os.ReadDir("/proc") if err != nil { @@ -68,6 +57,18 @@ func processGroupMatchesIdentity(pgid int, identity string) (bool, error) { return linuxProcessGroupMatchResult(pgid, scan.foundMember, scan.skippedStatErr) } +func linuxProcessGroupLeaderMismatch( + pgid int, + wanted linuxProcessTreeIdentity, +) (bool, error) { + fields, found, err := linuxProcessStat(pgid) + if err != nil || !found { + return false, err + } + _, mismatch, err := linuxProcessGroupFieldsMatch(pgid, pgid, wanted, fields) + return mismatch, err +} + type linuxProcessGroupScan struct { foundMember bool leaderMismatch bool diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index c9a813c5f2..e5a65a075f 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -114,6 +114,7 @@ func TestLinuxProcessGroupProbeHelper(t *testing.T) { func startLinuxProcessGroupHelper(t *testing.T) int { t.Helper() + // #nosec G204,G702 -- run this test binary with a fixed test selector. cmd := exec.Command(os.Args[0], "-test.run=^TestLinuxProcessGroupProbeHelper$") cmd.Env = append(os.Environ(), "DEVSY_PROCESS_GROUP_HELPER=1") cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} From 64aaee5336c97816dd23a7d7fc80ee4928157b87 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:07:04 -0600 Subject: [PATCH 35/49] fix(workspace): ignore canceled task status override --- pkg/client/clientimplementation/workspace_client.go | 3 ++- .../workspace_client_status_test.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 245fcad443..abbee315e0 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -17,6 +17,7 @@ import ( "github.com/devsy-org/devsy/pkg/agent" "github.com/devsy-org/devsy/pkg/agent/tunnelserver" "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/clierr" "github.com/devsy-org/devsy/pkg/compress" "github.com/devsy-org/devsy/pkg/config" config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" @@ -371,7 +372,7 @@ func (s *workspaceClient) taskStatusOverride() (client.Status, bool) { switch { case !latest.Status.Terminal(): return client.StatusProvisioning, true - case latest.Status == task.StatusFailed: + case latest.Status == task.StatusFailed && latest.ErrorCode != string(clierr.CodeCanceled): return client.StatusFailed, true default: return "", false diff --git a/pkg/client/clientimplementation/workspace_client_status_test.go b/pkg/client/clientimplementation/workspace_client_status_test.go index a93f77e32f..1ef5c53c8a 100644 --- a/pkg/client/clientimplementation/workspace_client_status_test.go +++ b/pkg/client/clientimplementation/workspace_client_status_test.go @@ -71,6 +71,18 @@ func TestTaskStatusOverride_FailedTaskReportsFailed(t *testing.T) { assertOverride(t, s, true, client.StatusFailed) } +func TestTaskStatusOverride_CanceledTaskDefersToWorkspaceStatus(t *testing.T) { + useTempTaskDir(t) + store, err := task.NewStore() + require.NoError(t, err) + tsk, err := store.Create(task.CreateOptions{Command: "up", WorkspaceID: testWorkspaceID}) + require.NoError(t, err) + require.NoError(t, tsk.Cancel()) + + s := &workspaceClient{providerWorkspace: &provider.Workspace{ID: testWorkspaceID}} + assertOverride(t, s, false, "") +} + func TestTaskStatusOverride_SucceededTaskDefersToContainerStatus(t *testing.T) { useTempTaskDir(t) store, err := task.NewStore() From fdc88e058a942b5f5a0554c7acfd2b67cf6ec73a Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:59:19 -0600 Subject: [PATCH 36/49] refactor(task): supervise detached worker lifecycle --- Taskfile.yml | 12 +- cmd/workspace/stop.go | 17 +- cmd/workspace/stop_test.go | 14 +- cmd/workspace/up/detach.go | 74 ++- .../clientimplementation/workspace_client.go | 5 +- pkg/command/background.go | 88 +++- pkg/command/job_windows.go | 12 + pkg/command/process.go | 36 ++ pkg/command/process_identity_darwin.go | 2 + pkg/command/process_identity_linux.go | 2 + pkg/command/process_identity_unix.go | 4 +- pkg/command/process_supported.go | 91 +++- pkg/command/process_windows.go | 87 ++++ pkg/command/process_windows_test.go | 39 +- pkg/config/pathmanager_unix.go | 74 +++ .../config/host_requirements_storage_unix.go | 9 +- pkg/task/store.go | 206 +++++--- pkg/task/store_test.go | 45 +- pkg/task/task.go | 457 +++++++++++++----- pkg/task/task_pid_supported_test.go | 18 +- pkg/task/task_pid_windows_test.go | 14 +- pkg/task/task_test.go | 280 +++++++++-- pkg/workspace/delete.go | 26 +- pkg/workspace/delete_test.go | 10 +- pkg/workspace/quiesce.go | 38 +- pkg/workspace/quiesce_test.go | 36 +- 26 files changed, 1331 insertions(+), 365 deletions(-) create mode 100644 pkg/config/pathmanager_unix.go diff --git a/Taskfile.yml b/Taskfile.yml index f4766c8dfa..f7dfeee7a2 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -116,10 +116,20 @@ tasks: host_os="$(go env GOOS)" host_arch="$(go env GOARCH)" host_binary="devsy-${host_os}-${host_arch}" - host_dir="dist/devsy-dev_${host_os}_${host_arch}_v1" if [ "$host_os" = "windows" ]; then host_binary="${host_binary}.exe" fi + host_dir="" + for candidate in dist/devsy-dev_${host_os}_${host_arch}_*; do + if [ -d "$candidate" ]; then + if [ -n "$host_dir" ]; then + echo "multiple host build directories match ${host_os}/${host_arch}" >&2 + exit 1 + fi + host_dir="$candidate" + fi + done + test -n "$host_dir" test -f "${host_dir}/${host_binary}" cp "${host_dir}/${host_binary}" "e2e/bin/${host_binary}" diff --git a/cmd/workspace/stop.go b/cmd/workspace/stop.go index 7d0287b3dc..bba8ec4cea 100644 --- a/cmd/workspace/stop.go +++ b/cmd/workspace/stop.go @@ -22,8 +22,7 @@ import ( type StopCmd struct { *flags.GlobalFlags client2.StopOptions - // Optional quiescence override for focused command tests. - quiesceUpTasks func(workspaceID string) error + quiescer workspace2.UpTaskQuiescer } // NewStopCmd creates a new destroy command. @@ -127,7 +126,7 @@ func (cmd *StopCmd) run( ) error { // Quiesce before waiting for the workspace lock: a still-provisioning up // worker can be the one holding it. - if err := cmd.quiesce(client.Workspace()); err != nil { + if err := cmd.quiesce(ctx, client.Workspace()); err != nil { return err } @@ -148,7 +147,7 @@ func (cmd *StopCmd) run( // Quiesce again under the lock: a task may have become visible while // the lock wait blocked, and a surviving up worker can restart the // workspace after the stop. - if err := cmd.quiesce(client.Workspace()); err != nil { + if err := cmd.quiesce(ctx, client.Workspace()); err != nil { return err } @@ -271,12 +270,12 @@ func otherWorkspaceUsesMachine( return false } -func (cmd *StopCmd) quiesce(workspaceID string) error { - quiesce := cmd.quiesceUpTasks - if quiesce == nil { - quiesce = workspace2.QuiesceUpTasksForWorkspace +func (cmd *StopCmd) quiesce(ctx context.Context, workspaceID string) error { + quiescer := cmd.quiescer + if quiescer == nil { + quiescer = workspace2.StoreUpTaskQuiescer{} } - if err := quiesce(workspaceID); err != nil { + if err := quiescer.Quiesce(ctx, workspaceID); err != nil { return fmt.Errorf( "cancel detached up tasks for workspace %s: %w", workspaceID, diff --git a/cmd/workspace/stop_test.go b/cmd/workspace/stop_test.go index c95bb0c789..5fd572793e 100644 --- a/cmd/workspace/stop_test.go +++ b/cmd/workspace/stop_test.go @@ -19,6 +19,12 @@ type stopRecordingClient struct { calls *[]string } +type quiescerFunc func(context.Context, string) error + +func (f quiescerFunc) Quiesce(ctx context.Context, workspaceID string) error { + return f(ctx, workspaceID) +} + func (c *stopRecordingClient) Stop(ctx context.Context, opts client2.StopOptions) error { *c.calls = append(*c.calls, "stop") return c.fakeWorkspaceClient.Stop(ctx, opts) @@ -42,11 +48,11 @@ func TestStopCancelsPersistedUpTaskBeforeProviderStop(t *testing.T) { } client := &stopRecordingClient{fakeWorkspaceClient: fake, calls: &calls} cmd := &StopCmd{GlobalFlags: &flags.GlobalFlags{ResultFormat: formatPlain}} - cmd.quiesceUpTasks = func(workspaceID string) error { + cmd.quiescer = quiescerFunc(func(_ context.Context, workspaceID string) error { assert.Equal(t, testWorkspaceName, workspaceID) calls = append(calls, "quiesce") return nil - } + }) require.NoError(t, cmd.run(t.Context(), newStopTestConfig(), client)) @@ -80,9 +86,9 @@ func TestStopDoesNotProceedWhenActiveTaskCannotBeKilled(t *testing.T) { var calls []string client := &stopRecordingClient{fakeWorkspaceClient: fake, calls: &calls} cmd := &StopCmd{GlobalFlags: &flags.GlobalFlags{ResultFormat: formatPlain}} - cmd.quiesceUpTasks = func(string) error { + cmd.quiescer = quiescerFunc(func(context.Context, string) error { return errors.New("boom") - } + }) err := cmd.run(t.Context(), newStopTestConfig(), client) require.Error(t, err) diff --git a/cmd/workspace/up/detach.go b/cmd/workspace/up/detach.go index e6aa154810..fb95442762 100644 --- a/cmd/workspace/up/detach.go +++ b/cmd/workspace/up/detach.go @@ -1,10 +1,12 @@ package up import ( + "errors" "fmt" "os" "os/exec" "strings" + "time" "github.com/devsy-org/devsy/pkg/command" config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" @@ -18,9 +20,6 @@ import ( // runDetached submits this invocation as a background task and returns // immediately. func (cmd *UpCmd) runDetached(args []string) error { - if _, err := command.ProcessTreeIdentity(os.Getpid()); err != nil { - return fmt.Errorf("detached up is unsupported on this platform: %w", err) - } store, err := task.NewStore() if err != nil { return err @@ -37,7 +36,12 @@ func (cmd *UpCmd) runDetached(args []string) error { return err } - if err := launchDetached(t.ID()); err != nil { + if err := t.BeginLaunch(); err != nil { + _ = t.Fail(err) + return fmt.Errorf("prepare detached up: %w", err) + } + if err := launchDetached(t); err != nil { + _ = t.FinishLaunch() _ = t.Fail(err) return fmt.Errorf("launch detached up: %w", err) } @@ -55,14 +59,19 @@ func (cmd *UpCmd) runDetached(args []string) error { return err } -func launchDetached(taskID string) error { +func launchDetached(t *task.Task) error { execPath, err := os.Executable() if err != nil { return err } - args := append(detachedArgs(os.Args[1:]), names.Flag(names.TaskID), taskID) - return command.StartBackground(task.WorkerProcessName(taskID), func() (*exec.Cmd, error) { + args := append(detachedArgs(os.Args[1:]), names.Flag(names.TaskID), t.ID()) + return command.StartSupervisedBackground(command.SupervisedStartOptions{ + Name: task.WorkerProcessName(t.ID()), + OnStarted: func(ref command.ProcessRef) error { + return t.SetProcess(ref) + }, + }, func() (*exec.Cmd, error) { return &exec.Cmd{ Path: execPath, Args: append([]string{execPath}, args...), @@ -125,26 +134,41 @@ func (cmd *UpCmd) openTask() (*task.Task, error) { failTask(t, err) return nil, err } - if err := t.SetPID(os.Getpid()); err != nil { - failTask(t, err) - _ = t.ReleaseWorkerLock() - return nil, err - } - state, err := store.Get(cmd.taskID) - if err != nil { - failTask(t, err) - _ = t.ReleaseWorkerLock() - return nil, err - } - // Canceled before this worker claimed its lock: the canceled state is - // already recorded and this worker must not overwrite it or run up. - if state.Status.Terminal() { - if err := t.ReleaseWorkerLock(); err != nil { - return nil, fmt.Errorf("release canceled worker lock: %w", err) + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + state, err := store.Get(cmd.taskID) + if err != nil { + failTask(t, err) + _ = t.ReleaseWorkerLock() + return nil, err + } + if state.Status.Terminal() || state.CancelRequested { + if err := t.ReleaseWorkerLock(); err != nil { + return nil, fmt.Errorf("release canceled worker lock: %w", err) + } + return nil, task.ErrCanceled + } + if !state.LaunchPending { + if _, ok := state.ProcessReference(); !ok { + err := errors.New("detached worker launch did not publish process metadata") + failTask(t, err) + _ = t.ReleaseWorkerLock() + return nil, err + } + return t, nil + } + select { + case <-deadline.C: + err := errors.New("timed out waiting for detached worker launch metadata") + failTask(t, err) + _ = t.ReleaseWorkerLock() + return nil, err + case <-ticker.C: } - return nil, task.ErrCanceled } - return t, nil } func failTask(t *task.Task, err error) { diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index abbee315e0..5a81aeefc6 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -17,7 +17,6 @@ import ( "github.com/devsy-org/devsy/pkg/agent" "github.com/devsy-org/devsy/pkg/agent/tunnelserver" "github.com/devsy-org/devsy/pkg/client" - "github.com/devsy-org/devsy/pkg/clierr" "github.com/devsy-org/devsy/pkg/compress" "github.com/devsy-org/devsy/pkg/config" config2 "github.com/devsy-org/devsy/pkg/devcontainer/config" @@ -372,7 +371,7 @@ func (s *workspaceClient) taskStatusOverride() (client.Status, bool) { switch { case !latest.Status.Terminal(): return client.StatusProvisioning, true - case latest.Status == task.StatusFailed && latest.ErrorCode != string(clierr.CodeCanceled): + case latest.Status == task.StatusFailed && !latest.Canceled(): return client.StatusFailed, true default: return "", false @@ -395,7 +394,7 @@ func (s *workspaceClient) latestUpTask() *task.State { if latest == nil { return nil } - return store.Reconcile(latest) + return store.ReconcileState(latest) } // newestUpTask picks the most recently started `up` task for workspaceID. diff --git a/pkg/command/background.go b/pkg/command/background.go index 383ea24ba6..bd40a55b59 100644 --- a/pkg/command/background.go +++ b/pkg/command/background.go @@ -13,6 +13,11 @@ import ( type CreateCommand func() (*exec.Cmd, error) +type SupervisedStartOptions struct { + Name string + OnStarted func(ProcessRef) error +} + // StartBackgroundOnce starts a background process, ensuring only one instance // with the given commandName runs at a time. If a process is already running // (determined by PID file), or the lock cannot be acquired, it returns nil. @@ -59,7 +64,7 @@ func StartBackgroundOnce(commandName string, createCommand CreateCommand) error return err } - return startCommand(cmd, commandName, paths.pidFile, paths.streamsFile) + return startCommand(cmd, paths.pidFile, paths.streamsFile) } type processPaths struct { @@ -98,32 +103,45 @@ func StartBackground(commandName string, createCommand CreateCommand) error { return err } - return startDetached(cmd, commandName, "", streamsFile) + return startDetached(cmd, "", streamsFile) +} + +// StartSupervisedBackground starts a detached worker with an owned process +// tree and publishes its reference before the launch is considered complete. +func StartSupervisedBackground(opts SupervisedStartOptions, createCommand CreateCommand) error { + if opts.Name == "" { + return errors.New("supervised process name is required") + } + if opts.OnStarted == nil { + return errors.New("supervised process publication callback is required") + } + streamsFile, err := config.DefaultPathManager().ProcessStreamsFile(opts.Name) + if err != nil { + return fmt.Errorf("process streams file: %w", err) + } + cmd, err := createCommand() + if err != nil { + return err + } + return startSupervisedDetached(cmd, opts, streamsFile) } -func startCommand(cmd *exec.Cmd, commandName, pidFile, streamsFile string) error { - return startDetached(cmd, commandName, pidFile, streamsFile) +func startCommand(cmd *exec.Cmd, pidFile, streamsFile string) error { + return startDetached(cmd, pidFile, streamsFile) } -func startDetached(cmd *exec.Cmd, commandName, pidFile, streamsFile string) error { +func startDetached(cmd *exec.Cmd, pidFile, streamsFile string) error { streamsF, err := openStreamsFile(cmd, streamsFile) if err != nil { return err } - prepareBackgroundTree(cmd) if err := cmd.Start(); err != nil { closeFile(streamsF) return fmt.Errorf("start process: %w", err) } closeFile(streamsF) - if err := ownProcessTree(cmd.Process.Pid, commandName); err != nil { - _ = cmd.Process.Kill() - _ = cmd.Wait() - return fmt.Errorf("assign process tree ownership for pid %d: %w", cmd.Process.Pid, err) - } - if pidFile != "" { if err := os.WriteFile(pidFile, []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { _ = cmd.Process.Kill() @@ -132,17 +150,47 @@ func startDetached(cmd *exec.Cmd, commandName, pidFile, streamsFile string) erro } } - if err := resumeBackgroundTree(cmd.Process.Pid); err != nil { - _ = cmd.Process.Kill() + _ = cmd.Process.Release() + + return nil +} + +func startSupervisedDetached(cmd *exec.Cmd, opts SupervisedStartOptions, streamsFile string) error { + streamsF, err := openStreamsFile(cmd, streamsFile) + if err != nil { + return err + } + prepareBackgroundTree(cmd) + if err := cmd.Start(); err != nil { + closeFile(streamsF) + return fmt.Errorf("start process: %w", err) + } + closeFile(streamsF) + pid := cmd.Process.Pid + cleanup := func() { + _ = abortSupervisedLaunch(pid, opts.Name) _ = cmd.Wait() - if pidFile != "" { - _ = os.Remove(pidFile) - } - return fmt.Errorf("resume process %d: %w", cmd.Process.Pid, err) } - + if err := ownProcessTree(pid, opts.Name); err != nil { + cleanup() + return fmt.Errorf("assign process tree ownership for pid %d: %w", pid, err) + } + ref, err := processRefForStartedProcess(pid, opts.Name) + if err != nil { + cleanup() + return fmt.Errorf("capture process reference for pid %d: %w", pid, err) + } + if err := opts.OnStarted(ref); err != nil { + _ = terminateProcessRef(ref) + cleanup() + return fmt.Errorf("publish process reference for pid %d: %w", pid, err) + } + if err := resumeBackgroundTree(pid); err != nil { + _ = terminateProcessRef(ref) + cleanup() + return fmt.Errorf("resume process %d: %w", pid, err) + } _ = cmd.Process.Release() - return nil } diff --git a/pkg/command/job_windows.go b/pkg/command/job_windows.go index 1de7d8aa87..57bdc4156f 100644 --- a/pkg/command/job_windows.go +++ b/pkg/command/job_windows.go @@ -269,6 +269,18 @@ func terminateJobAfterWorkerExit(workerName string) (bool, error) { return terminateJobHandle(job, workerName) } +func terminateNamedJob(jobID string) (bool, error) { + if jobID == "" { + return false, errors.New("worker job reference is empty") + } + job, found, err := openNamedJob(jobID, jobID) + if err != nil || !found { + return found, err + } + defer func() { _ = windows.CloseHandle(job) }() + return terminateJobHandle(job, jobID) +} + func isDetachedTaskWorkerName(workerName string) bool { if !strings.HasPrefix(workerName, detachedTaskWorkerPrefix) { return false diff --git a/pkg/command/process.go b/pkg/command/process.go index da16d63bf2..c58c6f3172 100644 --- a/pkg/command/process.go +++ b/pkg/command/process.go @@ -1,5 +1,41 @@ package command +// ProcessTreeKind identifies the platform primitive that owns a worker tree. +type ProcessTreeKind string + +const ( + ProcessTreeLegacyPID ProcessTreeKind = "legacy-pid" + ProcessTreeUnixGroup ProcessTreeKind = "unix-group-v1" + ProcessTreeWindowsJob ProcessTreeKind = "windows-job-v1" +) + +// ProcessRef is the persisted reference used to supervise a detached worker. +// TreeID and Identity are opaque outside this package. +type ProcessRef struct { + PID int `json:"pid,omitempty"` + TreeKind ProcessTreeKind `json:"treeKind,omitempty"` + TreeID string `json:"treeId,omitempty"` + Identity string `json:"identity,omitempty"` +} + +// ProcessController provides the two operations needed by task cancellation. +type ProcessController interface { + Terminate(ProcessRef) error + CleanupAfterExit(ProcessRef) error +} + +type processController struct{} + +func DefaultProcessController() ProcessController { return processController{} } + +func (processController) Terminate(ref ProcessRef) error { + return terminateProcessRef(ref) +} + +func (processController) CleanupAfterExit(ref ProcessRef) error { + return cleanupExitedProcessRef(ref) +} + func IsRunning(pid string) (bool, error) { return isRunning(pid) } diff --git a/pkg/command/process_identity_darwin.go b/pkg/command/process_identity_darwin.go index ff317d9a9e..17d0657dab 100644 --- a/pkg/command/process_identity_darwin.go +++ b/pkg/command/process_identity_darwin.go @@ -12,6 +12,8 @@ import ( "golang.org/x/sys/unix" ) +const strongProcessIdentitySupported = true + func processTreeIdentity(pid int) (string, error) { sessionID, err := darwinSessionID(pid) if err != nil { diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index 127ab57bfe..498c692098 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -13,6 +13,8 @@ import ( "syscall" ) +const strongProcessIdentitySupported = true + func processTreeIdentity(pid int) (string, error) { fields, ok, err := linuxProcessStat(pid) if err != nil { diff --git a/pkg/command/process_identity_unix.go b/pkg/command/process_identity_unix.go index ef795e2eb1..5298036421 100644 --- a/pkg/command/process_identity_unix.go +++ b/pkg/command/process_identity_unix.go @@ -4,8 +4,10 @@ package command import "errors" +const strongProcessIdentitySupported = false + func processTreeIdentity(_ int) (string, error) { - return "", errors.New("process tree identity is implemented on Linux, macOS, and Windows") + return "", nil } func processGroupMatchesIdentity(_ int, _ string) (bool, error) { diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index 4686fcd9be..4c8a67c516 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -28,18 +28,93 @@ func isRunning(pid string) (bool, error) { err = process.Signal(syscall.Signal(0)) if err != nil { - if errors.Is(err, os.ErrProcessDone) || errors.Is(err, syscall.ESRCH) { + if isExitOrPerm(err) || errors.Is(err, os.ErrProcessDone) { return false, nil } - if errors.Is(err, syscall.EPERM) { - return true, nil - } return false, fmt.Errorf("check process %d: %w", parsedPid, err) } return true, nil } +func processRefForStartedProcess(pid int, _ string) (ProcessRef, error) { + identity, err := processTreeIdentity(pid) + if err != nil { + if !strongProcessIdentitySupported { + identity = "" + } else { + return ProcessRef{}, err + } + } + return ProcessRef{ + PID: pid, + TreeKind: ProcessTreeUnixGroup, + TreeID: strconv.Itoa(pid), + Identity: identity, + }, nil +} + +func terminateProcessRef(ref ProcessRef) error { + if ref.PID <= 0 { + return fmt.Errorf("invalid worker PID %d", ref.PID) + } + if ref.TreeKind == ProcessTreeLegacyPID { + if ref.Identity != "" { + return killTreeWithIdentity(strconv.Itoa(ref.PID), ref.TreeID, ref.Identity) + } + return terminateWeakProcessGroup(ref.PID) + } + if ref.TreeKind != ProcessTreeUnixGroup { + return fmt.Errorf("unsupported Unix process tree kind %q", ref.TreeKind) + } + groupID := ref.TreeID + if groupID == "" { + groupID = strconv.Itoa(ref.PID) + } + if ref.Identity != "" { + return killTreeWithIdentity(groupID, "detached-task", ref.Identity) + } + return terminateWeakProcessGroup(ref.PID) +} + +func terminateWeakProcessGroup(pid int) error { + running, err := isRunning(strconv.Itoa(pid)) + if err != nil || !running { + return err + } + if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil && !isExitOrPerm(err) { + return fmt.Errorf("send SIGTERM to worker group %d: %w", pid, err) + } + return nil +} + +func cleanupExitedProcessRef(ref ProcessRef) error { + if ref.TreeKind == ProcessTreeLegacyPID { + if ref.Identity == "" { + return nil + } + return killTreeAfterWorkerExit(strconv.Itoa(ref.PID), ref.TreeID, ref.Identity) + } + if ref.TreeKind != ProcessTreeUnixGroup || ref.Identity == "" { + return nil + } + groupID := ref.TreeID + if groupID == "" { + groupID = strconv.Itoa(ref.PID) + } + return killTreeAfterWorkerExit(groupID, "detached-task", ref.Identity) +} + +func abortSupervisedLaunch(pid int, _ string) error { + for _, target := range []int{-pid, pid} { + if err := syscall.Kill(target, syscall.SIGKILL); err != nil && + !isExitOrPerm(err) { + return fmt.Errorf("terminate uncommitted worker %d: %w", pid, err) + } + } + return nil +} + // Detached workers use a session identity to distinguish their group after the leader exits. func killTree(pid, treeName string) error { parsedPid, err := strconv.Atoi(pid) @@ -156,9 +231,13 @@ func checkUnidentifiedProcessGroup(pid int, treeName string) error { ) } +func isExitOrPerm(err error) bool { + return err != nil && (errors.Is(err, syscall.ESRCH) || errors.Is(err, syscall.EPERM)) +} + func signalProcessTree(target int, graceful bool, identity string) error { if err := syscall.Kill(target, syscall.SIGTERM); err != nil { - if errors.Is(err, syscall.ESRCH) { + if isExitOrPerm(err) { return nil // already exited } return fmt.Errorf("send SIGTERM to process target %d: %w", target, err) @@ -174,7 +253,7 @@ func signalProcessTree(target int, graceful bool, identity string) error { return nil } err = syscall.Kill(target, syscall.SIGKILL) - if err != nil && !errors.Is(err, syscall.ESRCH) { + if err != nil && !isExitOrPerm(err) { return fmt.Errorf("send SIGKILL to process target %d: %w", target, err) } return nil diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index 12d5775b95..a027021c01 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -12,6 +12,8 @@ import ( "golang.org/x/sys/windows" ) +const strongProcessIdentitySupported = true + func isRunning(pid string) (bool, error) { parsed, err := parsePID(pid) if err != nil { @@ -47,6 +49,91 @@ func isRunning(pid string) (bool, error) { } } +func processRefForStartedProcess(pid int, workerName string) (ProcessRef, error) { + jobID, err := jobNameFor(workerName) + if err != nil { + return ProcessRef{}, err + } + identity, err := processTreeIdentity(pid) + if err != nil { + return ProcessRef{}, err + } + return ProcessRef{ + PID: pid, + TreeKind: ProcessTreeWindowsJob, + TreeID: jobID, + Identity: identity, + }, nil +} + +func terminateProcessRef(ref ProcessRef) error { + if ref.TreeKind == ProcessTreeWindowsJob { + terminated, err := terminateNamedJob(ref.TreeID) + if err != nil { + return err + } + if terminated { + return nil + } + running, err := isRunning(strconv.Itoa(ref.PID)) + if err != nil { + return err + } + if !running { + return nil + } + return fmt.Errorf( + "worker job %s is unavailable while process %d is running", + ref.TreeID, + ref.PID, + ) + } + if ref.TreeKind == ProcessTreeLegacyPID { + pid := strconv.Itoa(ref.PID) + if ref.Identity != "" { + return killTreeWithIdentity(pid, ref.TreeID, ref.Identity) + } + return killTree(pid, ref.TreeID) + } + return fmt.Errorf("unsupported Windows process tree kind %q", ref.TreeKind) +} + +func cleanupExitedProcessRef(ref ProcessRef) error { + if ref.TreeKind == ProcessTreeWindowsJob { + terminated, err := terminateNamedJob(ref.TreeID) + if err != nil || terminated { + return err + } + running, err := isRunning(strconv.Itoa(ref.PID)) + if err != nil { + return err + } + if running { + return fmt.Errorf( + "worker job %s is unavailable while process %d is running", + ref.TreeID, + ref.PID, + ) + } + return nil + } + if ref.TreeKind == ProcessTreeLegacyPID && ref.Identity != "" { + return killTreeAfterWorkerExit(strconv.Itoa(ref.PID), ref.TreeID, ref.Identity) + } + return nil +} + +func abortSupervisedLaunch(pid int, workerName string) error { + terminated, err := terminateJobForPID(workerName, pid) + if err != nil { + return err + } + if terminated { + return nil + } + return killTree(strconv.Itoa(pid), "") +} + func killTree(pid, treeName string) error { parsed, err := parsePID(pid) if err != nil { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index cb9423b5cf..4bd591f1cf 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -188,19 +188,26 @@ func TestIsRunningRecognizesExitCode259AsExited(t *testing.T) { } } -func TestStartDetachedAssignsWorkerToJobBeforeItRuns(t *testing.T) { +func TestStartSupervisedBackgroundAssignsWorkerToJobBeforeItRuns(t *testing.T) { dir := t.TempDir() + config.ResetPathManager() + t.Cleanup(config.ResetPathManager) + t.Setenv(config.EnvHome, dir) jobName := "devsy-test-suspended-worker" jobResult := filepath.Join(dir, "in-job") - pidFile := filepath.Join(dir, "worker.pid") cmd := exec.Command(os.Args[0], "-test.run", "TestHelperProcess") cmd.Env = helperEnv(helperEnvMarker+"=1", helperEnvCheckJob+"="+jobResult) - - if err := startDetached(cmd, jobName, pidFile, filepath.Join(dir, "streams")); err != nil { - t.Fatalf("startDetached: %v", err) - } - pid := waitForPIDFile(t, pidFile) - t.Cleanup(func() { _ = KillTree(strconv.Itoa(pid), jobName) }) + var ref ProcessRef + if err := startSupervisedDetached(cmd, SupervisedStartOptions{ + Name: jobName, + OnStarted: func(process ProcessRef) error { + ref = process + return nil + }, + }, filepath.Join(dir, "streams")); err != nil { + t.Fatalf("startSupervisedDetached: %v", err) + } + t.Cleanup(func() { _ = DefaultProcessController().Terminate(ref) }) job, found, err := openWorkerJob(jobName) if err != nil || !found { t.Fatalf("open worker job after launcher exit: found=%t, err=%v", found, err) @@ -595,15 +602,21 @@ func TestKillTerminatesOrphanedGrandchildViaJobObject(t *testing.T) { helperEnvChildPIDFile+"="+grandchildPIDFile, helperEnvExitAfterSpawn+"=1", ) - if err := startDetached( + var ref ProcessRef + if err := startSupervisedDetached( parent, - "devsy-test-worker", - parentPIDFile, + SupervisedStartOptions{ + Name: "devsy-test-worker", + OnStarted: func(process ProcessRef) error { + ref = process + return os.WriteFile(parentPIDFile, []byte(strconv.Itoa(process.PID)), 0o600) + }, + }, filepath.Join(dir, "streams"), ); err != nil { - t.Fatalf("startDetached: %v", err) + t.Fatalf("startSupervisedDetached: %v", err) } - parentPID := waitForPIDFile(t, parentPIDFile) + parentPID := ref.PID identity, err := ProcessTreeIdentity(parentPID) if err != nil { t.Fatalf("ProcessTreeIdentity: %v", err) diff --git a/pkg/config/pathmanager_unix.go b/pkg/config/pathmanager_unix.go new file mode 100644 index 0000000000..139dfd41f3 --- /dev/null +++ b/pkg/config/pathmanager_unix.go @@ -0,0 +1,74 @@ +//go:build unix && !linux && !darwin + +package config + +import ( + "fmt" + "os" + "path/filepath" +) + +type unixPathManager struct { + basePathManager +} + +func newPlatformPathManager() PathManager { + pm := &unixPathManager{} + pm.pm = pm + return pm +} + +func (u *unixPathManager) ConfigDir() (string, error) { + if dir, ok, err := homeOverrideDir(); ok { + return dir, err + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("config dir: %w", err) + } + return ensureDir(filepath.Join(home, "."+RepoName)) +} + +func (u *unixPathManager) DataDir() (string, error) { + if dir, ok, err := homeOverrideDir(); ok { + return dir, err + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("data dir: %w", err) + } + return ensureDir(filepath.Join(home, "."+RepoName)) +} + +func (u *unixPathManager) CacheDir() (string, error) { + if dir, ok, err := homeOverrideDir("cache"); ok { + return dir, err + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("cache dir: %w", err) + } + return ensureDir(filepath.Join(home, ".cache", RepoName)) +} + +func (u *unixPathManager) StateDir() (string, error) { + if dir, ok, err := homeOverrideDir("state"); ok { + return dir, err + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("state dir: %w", err) + } + return ensureDir(filepath.Join(home, "."+RepoName, "state")) +} + +func (u *unixPathManager) RuntimeDir() (string, error) { + if dir, ok, err := homeOverrideDir("run"); ok { + return dir, err + } + return ensureDir(filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d", RepoName, os.Getuid()))) +} + +func (u *unixPathManager) SystemBinDir() (string, error) { + return "/usr/local/bin", nil +} diff --git a/pkg/devcontainer/config/host_requirements_storage_unix.go b/pkg/devcontainer/config/host_requirements_storage_unix.go index ee8a23f3ab..33500622cd 100644 --- a/pkg/devcontainer/config/host_requirements_storage_unix.go +++ b/pkg/devcontainer/config/host_requirements_storage_unix.go @@ -9,5 +9,12 @@ func availableStorageBytes(path string) (uint64, error) { if err := syscall.Statfs(path, &stat); err != nil { return 0, err } - return stat.Bavail * uint64(stat.Bsize), nil //nolint:gosec // Bsize type varies by platform + if stat.Bavail <= 0 || stat.Bsize <= 0 { + return 0, nil + } + return uint64( + stat.Bavail, + ) * uint64( + stat.Bsize, + ), nil //nolint:gosec // Statfs field types vary by platform } diff --git a/pkg/task/store.go b/pkg/task/store.go index b00818cdd9..9d63dbc5b9 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "sort" - "strconv" "time" "github.com/devsy-org/devsy/pkg/command" @@ -19,16 +18,10 @@ import ( // lockTimeout bounds how long update waits to acquire a task's file lock. const lockTimeout = 5 * time.Second -// pidPublishTimeout bounds Cancel's wait for a starting worker to publish -// its PID. -const pidPublishTimeout = 5 * time.Second - // Store persists task state as one JSON file per task under dir. type Store struct { dir string - killProcessWithIdentity func(pid, treeName, identity string) error - killExitedWorkerTree func(pid, treeName, identity string) error - killLegacyWorkerTree func(pid, treeName string) error + processController command.ProcessController afterClaimHook func() afterCancelLockClaimedHook func() } @@ -46,10 +39,8 @@ func NewStoreAt(dir string) (*Store, error) { return nil, fmt.Errorf("create task dir: %w", err) } return &Store{ - dir: dir, - killProcessWithIdentity: command.KillTreeWithIdentity, - killExitedWorkerTree: command.KillTreeAfterWorkerExit, - killLegacyWorkerTree: command.KillTree, + dir: dir, + processController: command.DefaultProcessController(), }, nil } @@ -132,45 +123,69 @@ func (s *Store) Abandoned(state *State) bool { return true } -// Reconcile marks a task failed when its worker died without recording a -// result, so it stops being reported as still running. Returns the effective -// state, which is unchanged for live and already-terminal tasks. -func (s *Store) Reconcile(state *State) *State { - if state == nil || state.Status.Terminal() { - return state +// ReconcileTask cleans up a task whose worker exited without recording a +// result, then records it as abandoned. +func (s *Store) ReconcileTask(ctx context.Context, id string) (*State, error) { + if err := ctx.Err(); err != nil { + return nil, err } - - // Held across the whole transition: releasing before the write would let a - // new worker claim the lock and then be marked failed while running. - lock, ok := s.claimDeadWorkerLock(state.ID) + current, err := s.Get(id) + if err != nil { + return nil, err + } + if !needsReconciliation(current) { + return current, nil + } + lock, ok := s.claimDeadWorkerLock(id) if !ok { - return state + return current, nil } defer func() { _ = lock.Unlock() }() - if s.afterClaimHook != nil { s.afterClaimHook() } + return s.reconcileClaimedTask(ctx, id) +} - current, err := s.Get(state.ID) - if err != nil { +func needsReconciliation(state *State) bool { + return state != nil && !state.Status.Terminal() && !state.CancelRequested && + !state.LaunchPending +} + +// ReconcileState marks a task failed after its worker exits without a result. +// It never terminates processes and is safe to use from workspace status. +func (s *Store) ReconcileState(state *State) *State { + if !needsStateReconciliation(state) { return state } - if current.Status.Terminal() { - return current + lock, ok := s.claimDeadWorkerLock(state.ID) + if !ok { + return state } - if current.PID != 0 { - if err := s.killExitedWorkerTree( - strconv.Itoa(current.PID), - WorkerProcessName(current.ID), - current.ProcessTreeIdentity, - ); err != nil { + defer func() { _ = lock.Unlock() }() + current, err := s.Get(state.ID) + if err != nil || !needsStateReconciliation(current) { + if err == nil { return current } + return state + } + if hasProcessReference(current) { + return current } return s.failAbandoned(current) } +func needsStateReconciliation(state *State) bool { + return state != nil && !state.Status.Terminal() && !state.CancelRequested && + !state.LaunchPending +} + +func hasProcessReference(state *State) bool { + _, ok := state.ProcessReference() + return ok +} + // List returns every known task, most recently started first. func (s *Store) List() ([]*State, error) { entries, err := os.ReadDir(s.dir) @@ -197,23 +212,30 @@ func (s *Store) List() ([]*State, error) { return states, nil } +// ForWorkspace returns matching tasks, newest first, without changing state. +func (s *Store) ForWorkspace(workspaceID, command string) ([]*State, error) { + states, err := s.List() + if err != nil { + return nil, err + } + matching := make([]*State, 0, len(states)) + for _, state := range states { + if state.WorkspaceID == workspaceID && (command == "" || state.Command == command) { + matching = append(matching, state) + } + } + return matching, nil +} + // ActiveForWorkspace returns non-terminal tasks for a workspace, newest first. -// A non-empty command restricts the match. func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error) { - states, err := s.List() + states, err := s.ForWorkspace(workspaceID, command) if err != nil { return nil, err } active := make([]*State, 0, len(states)) for _, state := range states { - if state.WorkspaceID != workspaceID { - continue - } - if command != "" && state.Command != command { - continue - } - state = s.Reconcile(state) if state.Status.Terminal() { continue } @@ -222,26 +244,68 @@ func (s *Store) ActiveForWorkspace(workspaceID, command string) ([]*State, error return active, nil } -// awaitPID polls for the worker to publish its PID, which it does right -// after claiming the worker lock. A concurrent cancellation can commit the -// terminal state first, leaving nothing to wait for. -func (s *Store) awaitPID(id string, timeout time.Duration) (int, bool) { - deadline := time.Now().Add(timeout) +type WorkerObservation struct { + State *State + WorkerGone bool +} + +// WaitForWorkerObservation waits until process metadata is published, the +// task becomes terminal, or its worker lock becomes available. +func (s *Store) WaitForWorkerObservation( + ctx context.Context, + id string, +) (WorkerObservation, error) { + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() for { - state, err := s.Get(id) - if err == nil { - if state.PID != 0 { - return state.PID, false - } - if state.Status.Terminal() { - return 0, true - } + observation, ready, err := s.workerObservation(id) + if err != nil { + return WorkerObservation{}, err + } + if ready { + return observation, nil + } + select { + case <-ctx.Done(): + return WorkerObservation{}, ctx.Err() + case <-ticker.C: + } + } +} + +func (s *Store) workerObservation(id string) (WorkerObservation, bool, error) { + state, err := s.Get(id) + if err != nil { + return WorkerObservation{}, false, err + } + if state.Status.Terminal() || hasProcessReference(state) { + return WorkerObservation{State: state}, true, nil + } + lock, available, err := s.tryWorkerLock(id) + if err != nil { + return WorkerObservation{}, false, err + } + if !available { + return WorkerObservation{}, false, nil + } + _ = lock.Unlock() + return WorkerObservation{State: state, WorkerGone: true}, true, nil +} + +func (s *Store) reconcileClaimedTask(ctx context.Context, id string) (*State, error) { + current, err := s.Get(id) + if err != nil || !needsReconciliation(current) { + return current, err + } + if ref, ok := current.ProcessReference(); ok { + if err := ctx.Err(); err != nil { + return current, err } - if time.Now().After(deadline) { - return 0, false + if err := s.processController.CleanupAfterExit(ref); err != nil { + return current, fmt.Errorf("clean up task %s process tree: %w", id, err) } - time.Sleep(50 * time.Millisecond) } + return s.failAbandoned(current), nil } // failAbandoned records ErrAbandoned for a task whose worker is confirmed gone. @@ -282,6 +346,22 @@ func (s *Store) tryWorkerLock(id string) (*flock.Flock, bool, error) { return lock, true, nil } +func (s *Store) tryLauncherLock(id string) (*flock.Flock, bool, error) { + path, err := s.launcherLockPath(id) + if err != nil { + return nil, false, err + } + lock := flock.New(path) + locked, err := lock.TryLock() + if err != nil { + return nil, false, err + } + if !locked { + return nil, false, nil + } + return lock, true, nil +} + // claimDeadWorkerLock acquires the task's worker lock, which only succeeds when // no worker holds it. func (s *Store) claimDeadWorkerLock(id string) (*flock.Flock, bool) { @@ -313,6 +393,14 @@ func (s *Store) workerLockPath(id string) (string, error) { return path + ".worker.lock", nil } +func (s *Store) launcherLockPath(id string) (string, error) { + path, err := s.path(id) + if err != nil { + return "", err + } + return path + ".launcher.lock", nil +} + func (s *Store) update(id string, mutate func(*State)) error { return s.withLock(id, func() error { state, err := s.Get(id) diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index 7688ab4968..d73701ce05 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -4,6 +4,8 @@ import ( "errors" "testing" "time" + + "github.com/devsy-org/devsy/pkg/command" ) const workspaceOne = "ws-1" @@ -69,7 +71,7 @@ func TestActiveForWorkspaceOrdersNewestFirst(t *testing.T) { } } -func TestActiveForWorkspaceReconcilesAbandonedWorkers(t *testing.T) { +func TestActiveForWorkspaceDoesNotReconcileAbandonedWorkers(t *testing.T) { store := newTestStore(t) tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) @@ -86,20 +88,20 @@ func TestActiveForWorkspaceReconcilesAbandonedWorkers(t *testing.T) { if err != nil { t.Fatalf("ActiveForWorkspace: %v", err) } - if len(active) != 0 { - t.Errorf("abandoned task reported active: %+v", active) + if len(active) != 1 || active[0].ID != tk.ID() { + t.Errorf("query omitted unreconciled task: %+v", active) } state, err := store.Get(tk.ID()) if err != nil { t.Fatalf("Get: %v", err) } - if !state.Status.Terminal() { - t.Errorf("reconciled task status = %q, want terminal", state.Status) + if state.Status.Terminal() { + t.Errorf("query changed task status to %q", state.Status) } } -func TestActiveForWorkspaceCleansAbandonedProcessTreeBeforeExcluding(t *testing.T) { +func TestActiveForWorkspaceDoesNotCleanAbandonedProcessTree(t *testing.T) { store := newTestStore(t) state := abandonTask(t, store) if err := store.Open(state.ID).SetWorkspaceID(workspaceOne); err != nil { @@ -110,30 +112,24 @@ func TestActiveForWorkspaceCleansAbandonedProcessTreeBeforeExcluding(t *testing. } killCalls := 0 - setKillProcessForTest(store, func(pid, treeName string) error { - killCalls++ - if pid != "4242" { - t.Errorf("kill pid = %q, want 4242", pid) - } - if treeName != WorkerProcessName(state.ID) { - t.Errorf("kill tree = %q, want %q", treeName, WorkerProcessName(state.ID)) - } - return nil - }) + store.processController = fakeProcessController{ + terminate: func(command.ProcessRef) error { killCalls++; return nil }, + cleanup: func(command.ProcessRef) error { killCalls++; return nil }, + } active, err := store.ActiveForWorkspace(workspaceOne, "up") if err != nil { t.Fatalf("ActiveForWorkspace: %v", err) } - if len(active) != 0 { - t.Fatalf("abandoned task remains active after cleanup: %+v", active) + if len(active) != 1 || active[0].ID != state.ID { + t.Fatalf("query omitted an unreconciled task: %+v", active) } - if killCalls != 1 { - t.Errorf("process-tree cleanup called %d times, want 1", killCalls) + if killCalls != 0 { + t.Errorf("query invoked process cleanup %d times", killCalls) } } -func TestActiveForWorkspaceKeepsAbandonedTaskActiveWhenTreeCleanupFails(t *testing.T) { +func TestActiveForWorkspaceDoesNotMutateTaskWhenTreeCleanupWouldFail(t *testing.T) { store := newTestStore(t) state := abandonTask(t, store) if err := store.Open(state.ID).SetWorkspaceID(workspaceOne); err != nil { @@ -142,9 +138,10 @@ func TestActiveForWorkspaceKeepsAbandonedTaskActiveWhenTreeCleanupFails(t *testi if err := setPIDForTest(store.Open(state.ID)); err != nil { t.Fatalf("SetPID: %v", err) } - setKillProcessForTest(store, func(string, string) error { - return errors.New("descendant cleanup failed") - }) + store.processController = fakeProcessController{ + terminate: func(command.ProcessRef) error { return errors.New("unexpected terminate") }, + cleanup: func(command.ProcessRef) error { return errors.New("descendant cleanup failed") }, + } active, err := store.ActiveForWorkspace(workspaceOne, "up") if err != nil { diff --git a/pkg/task/task.go b/pkg/task/task.go index 7af3df7387..598f797fca 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -3,11 +3,11 @@ package task import ( + "context" "encoding/json" "errors" "fmt" "os" - "strconv" "time" "github.com/devsy-org/devsy/pkg/clierr" @@ -46,24 +46,27 @@ func (s Status) Terminal() bool { // doing the work, distinct from whatever process is merely polling this // state. type State struct { - ID string `json:"id"` - Command string `json:"command,omitempty"` - WorkspaceID string `json:"workspaceId,omitempty"` - Status Status `json:"status"` - Phase string `json:"phase,omitempty"` - Step string `json:"step,omitempty"` - OperationID string `json:"operationId,omitempty"` - ParentOperationID string `json:"parentOperationId,omitempty"` - DurationMs int64 `json:"durationMs,omitempty"` - Error string `json:"error,omitempty"` - ErrorCode string `json:"errorCode,omitempty"` - ErrorHint string `json:"errorHint,omitempty"` - ErrorContext map[string]string `json:"errorContext,omitempty"` - Result *config.Result `json:"result,omitempty"` - PID int `json:"pid,omitempty"` - ProcessTreeIdentity string `json:"processTreeIdentity,omitempty"` - StartedAt time.Time `json:"startedAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + Command string `json:"command,omitempty"` + WorkspaceID string `json:"workspaceId,omitempty"` + Status Status `json:"status"` + Phase string `json:"phase,omitempty"` + Step string `json:"step,omitempty"` + OperationID string `json:"operationId,omitempty"` + ParentOperationID string `json:"parentOperationId,omitempty"` + DurationMs int64 `json:"durationMs,omitempty"` + Error string `json:"error,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorHint string `json:"errorHint,omitempty"` + ErrorContext map[string]string `json:"errorContext,omitempty"` + Result *config.Result `json:"result,omitempty"` + PID int `json:"pid,omitempty"` + ProcessTreeIdentity string `json:"processTreeIdentity,omitempty"` + Process *command.ProcessRef `json:"process,omitempty"` + CancelRequested bool `json:"cancelRequested,omitempty"` + LaunchPending bool `json:"launchPending,omitempty"` + StartedAt time.Time `json:"startedAt"` + UpdatedAt time.Time `json:"updatedAt"` } // CreateOptions labels a task at creation time for later listing. @@ -79,19 +82,93 @@ type Task struct { store *Store id string // Held for the worker process's lifetime; see HoldWorkerLock. - workerLock *flock.Flock + workerLock *flock.Flock + launcherLock *flock.Flock } func (t *Task) ID() string { return t.id } +func (s *State) Canceled() bool { + return s != nil && s.Status == StatusFailed && s.ErrorCode == string(clierr.CodeCanceled) +} + +func (s *State) ProcessReference() (command.ProcessRef, bool) { + if s == nil { + return command.ProcessRef{}, false + } + if s.Process != nil { + if s.Process.PID > 0 { + return *s.Process, true + } + } + if s.PID <= 0 { + return command.ProcessRef{}, false + } + return command.ProcessRef{ + PID: s.PID, + TreeKind: command.ProcessTreeLegacyPID, + TreeID: WorkerProcessName(s.ID), + Identity: s.ProcessTreeIdentity, + }, true +} + +func (t *Task) BeginLaunch() error { + launcherLock, available, err := t.store.tryLauncherLock(t.id) + if err != nil { + return fmt.Errorf("lock task %s launcher: %w", t.id, err) + } + if !available { + return fmt.Errorf("task %s already has a launcher", t.id) + } + var canceled bool + err = t.store.update(t.id, func(s *State) { + if s.Status.Terminal() || s.CancelRequested { + canceled = true + return + } + s.LaunchPending = true + }) + if err != nil { + _ = launcherLock.Unlock() + return err + } + if canceled { + _ = launcherLock.Unlock() + return ErrCanceled + } + t.launcherLock = launcherLock + return nil +} + +func (t *Task) FinishLaunch() error { + err := t.store.update(t.id, func(s *State) { s.LaunchPending = false }) + return errors.Join(err, releaseLauncherLock(t)) +} + +func (t *Task) SetProcess(ref command.ProcessRef) error { + if ref.PID <= 0 { + return fmt.Errorf("invalid worker PID %d", ref.PID) + } + err := t.store.update(t.id, func(s *State) { + s.Process = &ref + s.PID = ref.PID + s.ProcessTreeIdentity = ref.Identity + s.LaunchPending = false + }) + return errors.Join(err, releaseLauncherLock(t)) +} + +// SetPID reads legacy task state and tests that publish a worker after launch. func (t *Task) SetPID(pid int) error { identity, err := command.ProcessTreeIdentity(pid) if err != nil { return fmt.Errorf("identify process tree for pid %d: %w", pid, err) } - return t.store.update(t.id, func(s *State) { - s.PID = pid - s.ProcessTreeIdentity = identity + return t.SetProcess(command.ProcessRef{ + PID: pid, + TreeKind: command.ProcessTreeLegacyPID, + TreeID: WorkerProcessName(t.id), + Identity: identity, }) } @@ -131,6 +208,18 @@ func (t *Task) ReleaseWorkerLock() error { return nil } +func releaseLauncherLock(t *Task) error { + if t.launcherLock == nil { + return nil + } + lock := t.launcherLock + t.launcherLock = nil + if err := lock.Unlock(); err != nil { + return fmt.Errorf("unlock task %s launcher: %w", t.id, err) + } + return nil +} + // SetWorkspaceID corrects the task's workspace label to the resolved ID, // which may differ from whatever label it was created with (e.g. a raw // source string guessed before workspace resolution ran). client.Status @@ -149,7 +238,7 @@ func (t *Task) Reporter() status.Reporter { // concurrently with a Cancel can't overwrite the canceled state with success. func (t *Task) Succeed(result *config.Result) error { return t.store.update(t.id, func(s *State) { - if s.Status.Terminal() { + if s.Status.Terminal() || s.CancelRequested { return } s.Status = StatusSucceeded @@ -164,46 +253,218 @@ func (t *Task) Succeed(result *config.Result) error { // Cancel terminates the worker before recording cancellation. A termination // failure leaves the task retryable. func (t *Task) Cancel() error { - var pid int - var identity string - observedNonterminal := false - err := t.store.update(t.id, func(s *State) { - if s.Status.Terminal() { - return - } - observedNonterminal = true - pid = s.PID - identity = s.ProcessTreeIdentity - }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return t.CancelContext(ctx) +} + +func (t *Task) CancelContext(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + state, err := t.requestCancel() if err != nil { return err } - if !observedNonterminal { + if state.Status.Terminal() { return nil } + return quiesceCancellation(t, ctx) +} - if pid == 0 { - var finalized bool - pid, finalized, err = t.cancelWithoutPublishedPID() +func quiesceCancellation(t *Task, ctx context.Context) error { + var terminated *command.ProcessRef + for { + retry, newTerminated, err := quiesceCancellationStep(t, ctx, terminated) if err != nil { return err } - if finalized { + if !retry { + return nil + } + terminated = newTerminated + if err := waitForCancellation(ctx); err != nil { + return fmt.Errorf("cancel task %s: %w", t.id, err) + } + } +} + +func quiesceCancellationStep( + t *Task, + ctx context.Context, + terminated *command.ProcessRef, +) (bool, *command.ProcessRef, error) { + state, err := t.store.Get(t.id) + if err != nil { + return false, nil, fmt.Errorf("cancel task %s: read state: %w", t.id, err) + } + if state.Status.Terminal() { + return false, nil, nil + } + finalized, err := tryFinalizeWithoutWorker(t, ctx) + if err != nil { + return false, nil, err + } + if finalized { + return false, nil, nil + } + + retry, ref, err := processCancellationObservation(t, state, terminated, ctx) + if err != nil { + return false, nil, err + } + if retry { + return true, terminated, nil + } + if ref != nil { + terminated = ref + } + return true, terminated, nil +} + +func processCancellationObservation( + t *Task, + state *State, + terminated *command.ProcessRef, + ctx context.Context, +) (bool, *command.ProcessRef, error) { + ref, hasRef := state.ProcessReference() + if hasRef { + if err := terminateWorker(t, ref, terminated, ctx); err != nil { + return false, nil, err + } + return false, &ref, nil + } + observation, err := t.store.WaitForWorkerObservation(ctx, t.id) + if err != nil { + return false, nil, fmt.Errorf("cancel task %s: wait for worker startup: %w", t.id, err) + } + return observation.WorkerGone, nil, nil +} + +func tryFinalizeWithoutWorker(t *Task, ctx context.Context) (bool, error) { + workerLock, available, err := t.store.tryWorkerLock(t.id) + if err != nil { + return false, fmt.Errorf("cancel task %s: inspect worker lock: %w", t.id, err) + } + if !available { + return false, nil + } + if t.store.afterCancelLockClaimedHook != nil { + t.store.afterCancelLockClaimedHook() + } + return true, cancelWithoutWorker(t, ctx, workerLock) +} + +func terminateWorker( + t *Task, + ref command.ProcessRef, + terminated *command.ProcessRef, + ctx context.Context, +) error { + if terminated != nil && *terminated == ref { + return nil + } + if err := t.store.processController.Terminate(ref); err != nil { + finalized, lockErr := tryFinalizeWithoutWorker(t, ctx) + if lockErr == nil && finalized { return nil } - state, err := t.store.Get(t.id) + return fmt.Errorf("cancel task %s: terminate worker tree: %w", t.id, err) + } + return nil +} + +func cancelWithoutWorker(t *Task, ctx context.Context, workerLock *flock.Flock) error { + defer func() { _ = workerLock.Unlock() }() + state, err := waitForLaunch(t, ctx) + if err != nil { + return err + } + if state.Status.Terminal() { + return nil + } + if ref, ok := state.ProcessReference(); ok { + if err := t.store.processController.CleanupAfterExit(ref); err != nil { + return fmt.Errorf("cancel task %s: clean up worker tree: %w", t.id, err) + } + } + return t.finalizeCanceled() +} + +func waitForLaunch(t *Task, ctx context.Context) (*State, error) { + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + state, done, err := waitForLaunchStep(t) if err != nil { - return fmt.Errorf("cancel task %s: read process tree identity: %w", t.id, err) + return nil, err + } + if done { + return state, nil + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf( + "cancel task %s: wait for launch publication: %w", + t.id, + ctx.Err(), + ) + case <-ticker.C: + } + } +} + +func waitForLaunchStep(t *Task) (*State, bool, error) { + state, err := t.store.Get(t.id) + if err != nil { + return nil, false, fmt.Errorf("cancel task %s: read state: %w", t.id, err) + } + if state.Status.Terminal() || !state.LaunchPending { + return state, true, nil + } + resolved, err := clearAbandonedLaunch(t) + if err != nil { + return nil, false, err + } + if resolved { + return nil, false, nil + } + return nil, false, nil +} + +func clearAbandonedLaunch(t *Task) (bool, error) { + launcherLock, available, err := t.store.tryLauncherLock(t.id) + if err != nil { + return false, fmt.Errorf("cancel task %s: inspect launcher lock: %w", t.id, err) + } + if !available { + return false, nil + } + defer func() { _ = launcherLock.Unlock() }() + err = t.store.update(t.id, func(current *State) { + if current.CancelRequested && current.LaunchPending { + current.LaunchPending = false } - identity = state.ProcessTreeIdentity + }) + if err != nil { + return false, fmt.Errorf("cancel task %s: clear abandoned launch: %w", t.id, err) } + return true, nil +} - if pid != 0 { - return t.cancelPublishedPID(pid, identity) +func waitForCancellation(ctx context.Context) error { + timer := time.NewTimer(20 * time.Millisecond) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil } - // This invocation observed the task as nonterminal, so a result the worker - // committed while unwinding does not win over the accepted cancellation. - return t.store.update(t.id, markCanceled) } func markCanceled(s *State) { @@ -233,7 +494,7 @@ func (t *Task) Fail(err error) error { } redactor := secrets.NewEnvironmentRedactor(os.Environ()) return t.store.update(t.id, func(s *State) { - if s.Status.Terminal() { + if s.Status.Terminal() || s.CancelRequested { return } s.Status = StatusFailed @@ -247,90 +508,28 @@ func (t *Task) Fail(err error) error { }) } -func (t *Task) cancelPublishedPID(pid int, identity string) error { - workerLock, locked, err := t.store.tryWorkerLock(t.id) - if err != nil { - return fmt.Errorf("cancel task %s: claim worker lock: %w", t.id, err) - } - if locked { - defer func() { _ = workerLock.Unlock() }() - return t.cancelExitedWorkerTree(pid) - } - return t.cancelLiveWorker(pid, identity) -} - -func (t *Task) cancelExitedWorkerTree(pid int) error { - state, err := t.store.Get(t.id) - if err != nil { - return fmt.Errorf("cancel task %s: read process tree identity: %w", t.id, err) - } - if err := t.store.killExitedWorkerTree( - strconv.Itoa(pid), - WorkerProcessName(t.id), - state.ProcessTreeIdentity, - ); err != nil { - return fmt.Errorf("cancel task %s: terminate worker tree: %w", t.id, err) - } - return t.store.update(t.id, markCanceled) -} - -func (t *Task) cancelLiveWorker(pid int, identity string) error { - // The held worker lock ties a legacy PID to its active worker. - pidString := strconv.Itoa(pid) - workerName := WorkerProcessName(t.id) - var err error - if identity == "" { - err = t.store.killLegacyWorkerTree(pidString, workerName) - } else { - err = t.store.killProcessWithIdentity(pidString, workerName, identity) - } - if err != nil { - workerLock, locked, lockErr := t.store.tryWorkerLock(t.id) - if lockErr != nil { - return fmt.Errorf( - "cancel task %s: terminate worker pid %d: %w (recheck worker lock: %v)", - t.id, - pid, - err, - lockErr, - ) - } - if !locked { - return fmt.Errorf("cancel task %s: terminate worker pid %d: %w", t.id, pid, err) +func (t *Task) requestCancel() (*State, error) { + var snapshot *State + err := t.store.update(t.id, func(state *State) { + if !state.Status.Terminal() { + state.CancelRequested = true } - defer func() { _ = workerLock.Unlock() }() - return t.cancelExitedWorkerTree(pid) + copy := *state + snapshot = © + }) + if err != nil { + return nil, err } - return t.store.update(t.id, markCanceled) + return snapshot, nil } -func (t *Task) cancelWithoutPublishedPID() (int, bool, error) { - workerLock, locked, err := t.store.tryWorkerLock(t.id) - if err != nil { - return 0, false, fmt.Errorf("cancel task %s: claim worker lock: %w", t.id, err) - } - if locked { - defer func() { _ = workerLock.Unlock() }() - if t.store.afterCancelLockClaimedHook != nil { - t.store.afterCancelLockClaimedHook() +func (t *Task) finalizeCanceled() error { + return t.store.update(t.id, func(state *State) { + if !state.Status.Terminal() && state.CancelRequested { + markCanceled(state) + state.LaunchPending = false } - // Holding the worker lock through the state update prevents a - // not-yet-started worker from passing its terminal-state check - // between our liveness check and cancellation commit. - return 0, true, t.store.update(t.id, markCanceled) - } - - pid, terminal := t.store.awaitPID(t.id, pidPublishTimeout) - if terminal { - return 0, true, nil - } - if pid == 0 { - return 0, false, fmt.Errorf( - "cancel task %s: worker has not published its pid yet, retry", - t.id, - ) - } - return pid, false, nil + }) } type taskReporter struct { @@ -342,7 +541,7 @@ func (r taskReporter) Report(e status.Event) { _ = r.task.store.update(r.task.id, func(s *State) { // A terminal task is done being described; late events from a worker // still unwinding must not resurrect it or rewrite its outcome. - if s.Status.Terminal() { + if s.Status.Terminal() || s.CancelRequested { return } if e.State == status.StateFailed { diff --git a/pkg/task/task_pid_supported_test.go b/pkg/task/task_pid_supported_test.go index b46be4378e..4faa282a96 100644 --- a/pkg/task/task_pid_supported_test.go +++ b/pkg/task/task_pid_supported_test.go @@ -22,7 +22,17 @@ func TestCancelSignalsALiveWorkersPID(t *testing.T) { if err := worker.Start(); err != nil { t.Fatalf("start: %v", err) } - defer func() { _ = worker.Process.Kill() }() + workerExited := make(chan struct{}) + var workerWaitErr error + go func() { + workerWaitErr = worker.Wait() + _ = tk.ReleaseWorkerLock() + close(workerExited) + }() + t.Cleanup(func() { + _ = worker.Process.Kill() + <-workerExited + }) if err := tk.SetPID(worker.Process.Pid); err != nil { t.Fatalf("SetPID: %v", err) } @@ -31,8 +41,8 @@ func TestCancelSignalsALiveWorkersPID(t *testing.T) { t.Fatalf("Cancel: %v", err) } - waitErr := worker.Wait() - if waitErr == nil { + <-workerExited + if workerWaitErr == nil { t.Fatal("worker process exited cleanly, want it signaled by Cancel") } } @@ -45,7 +55,7 @@ func TestCancelUsesPublishedProcessIdentity(t *testing.T) { } var gotIdentity string - setKillProcessWithIdentityForTest(store, func(_, _, identity string) error { + setKillProcessWithIdentityForTest(store, tk, func(_, _, identity string) error { gotIdentity = identity return nil }) diff --git a/pkg/task/task_pid_windows_test.go b/pkg/task/task_pid_windows_test.go index 158f53b0ba..7e8cec6a94 100644 --- a/pkg/task/task_pid_windows_test.go +++ b/pkg/task/task_pid_windows_test.go @@ -25,20 +25,20 @@ func TestCancelSignalsALiveWorkersPID(t *testing.T) { if err != nil { t.Fatalf("Create: %v", err) } - holdWorkerLock(t, tk) - worker := sleepWorker() workerName := WorkerProcessName(tk.ID()) - if err := command.StartBackground(workerName, func() (*exec.Cmd, error) { + if err := command.StartSupervisedBackground(command.SupervisedStartOptions{ + Name: workerName, + OnStarted: func(ref command.ProcessRef) error { + return tk.SetProcess(ref) + }, + }, func() (*exec.Cmd, error) { return worker, nil }); err != nil { - t.Fatalf("start background worker: %v", err) + t.Fatalf("start supervised worker: %v", err) } pid := worker.Process.Pid t.Cleanup(func() { _ = command.KillTree(strconv.Itoa(pid), workerName) }) - if err := tk.SetPID(pid); err != nil { - t.Fatalf("SetPID: %v", err) - } if err := tk.Cancel(); err != nil { t.Fatalf("Cancel: %v", err) diff --git a/pkg/task/task_test.go b/pkg/task/task_test.go index 0dd1f37485..1340efe92c 100644 --- a/pkg/task/task_test.go +++ b/pkg/task/task_test.go @@ -1,15 +1,18 @@ package task import ( + "context" "errors" "fmt" "os" + "strconv" "strings" "sync" "testing" "time" "github.com/devsy-org/devsy/pkg/clierr" + "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/status" "github.com/gofrs/flock" @@ -343,6 +346,106 @@ func TestSetPIDPersists(t *testing.T) { if state.PID != os.Getpid() { t.Errorf("PID = %d, want %d", state.PID, os.Getpid()) } + if state.Process == nil || state.Process.PID != os.Getpid() || + state.Process.TreeKind != command.ProcessTreeLegacyPID { + t.Errorf("process reference = %+v, want legacy PID reference", state.Process) + } +} + +func TestCancelIntentWaitsForProcessPublicationAndCleanup(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := tk.BeginLaunch(); err != nil { + t.Fatalf("BeginLaunch: %v", err) + } + + cleaned := make(chan command.ProcessRef, 1) + store.processController = fakeProcessController{ + cleanup: func(ref command.ProcessRef) error { + cleaned <- ref + return nil + }, + } + cancelDone := make(chan error, 1) + go func() { cancelDone <- tk.Cancel() }() + waitForCancelRequest(t, store, tk.ID()) + + assertCancelIntent(t, store, tk.ID()) + if err := tk.Succeed(&config.Result{}); err != nil { + t.Fatalf("Succeed during cancellation: %v", err) + } + if err := tk.SetProcess(command.ProcessRef{ + PID: 4242, + TreeKind: command.ProcessTreeUnixGroup, + TreeID: "4242", + }); err != nil { + t.Fatalf("SetProcess: %v", err) + } + + assertProcessCleaned(t, cleaned, cancelDone) + assertTaskCanceledResult(t, store, tk.ID()) +} + +func assertCancelIntent(t *testing.T, store *Store, id string) { + state, err := store.Get(id) + if err != nil { + t.Fatalf("Get pending cancellation: %v", err) + } + if state.Status.Terminal() || !state.CancelRequested { + t.Fatalf("cancel intent state = %+v, want non-terminal requested state", state) + } +} + +func assertProcessCleaned(t *testing.T, cleaned chan command.ProcessRef, cancelDone chan error) { + select { + case ref := <-cleaned: + if ref.PID != 4242 || ref.TreeID != "4242" { + t.Fatalf("cleaned process = %+v", ref) + } + case <-time.After(time.Second): + t.Fatal("cancellation finalized without process-tree cleanup") + } + if err := <-cancelDone; err != nil { + t.Fatalf("Cancel: %v", err) + } +} + +func assertTaskCanceledResult(t *testing.T, store *Store, id string) { + state, err := store.Get(id) + if err != nil { + t.Fatalf("Get canceled task: %v", err) + } + if !state.Canceled() || state.Result != nil { + t.Fatalf("final cancellation state = %+v", state) + } +} + +func TestCancelFinalizesLaunchAbandonedByLauncher(t *testing.T) { + store := newTestStore(t) + tk, err := store.Create(CreateOptions{}) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := tk.BeginLaunch(); err != nil { + t.Fatalf("BeginLaunch: %v", err) + } + if err := releaseLauncherLock(tk); err != nil { + t.Fatalf("release abandoned launcher lock: %v", err) + } + + if err := tk.Cancel(); err != nil { + t.Fatalf("Cancel: %v", err) + } + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if !state.Canceled() || state.LaunchPending { + t.Fatalf("abandoned launch state = %+v", state) + } } func TestCancelWithoutPIDMarksFailed(t *testing.T) { @@ -423,6 +526,28 @@ func assertWorkerLock(t *testing.T, lock *flock.Flock, wantLocked bool, when str } } +func waitForCancelRequest(t *testing.T, store *Store, id string) { + t.Helper() + deadline := time.NewTimer(time.Second) + ticker := time.NewTicker(time.Millisecond) + defer deadline.Stop() + defer ticker.Stop() + for { + state, err := store.Get(id) + if err != nil { + t.Fatalf("Get: %v", err) + } + if state.CancelRequested { + return + } + select { + case <-deadline.C: + t.Fatal("cancellation request was not persisted") + case <-ticker.C: + } + } +} + func TestOverlappingCancelsBeforePIDBothSucceed(t *testing.T) { store := newTestStore(t) tk, err := store.Create(CreateOptions{}) @@ -439,10 +564,19 @@ func TestOverlappingCancelsBeforePIDBothSucceed(t *testing.T) { firstDone := make(chan error, 1) go func() { firstDone <- tk.Cancel() }() <-lockClaimed + firstState, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get first cancellation state: %v", err) + } secondDone := make(chan error, 1) - go func() { secondDone <- tk.Cancel() }() - time.Sleep(200 * time.Millisecond) + secondStarted := make(chan struct{}) + go func() { + close(secondStarted) + secondDone <- tk.Cancel() + }() + <-secondStarted + waitForStateUpdate(t, store, tk.ID(), firstState) close(continueCancel) if err := <-firstDone; err != nil { @@ -454,6 +588,27 @@ func TestOverlappingCancelsBeforePIDBothSucceed(t *testing.T) { requireCanceledState(t, store, tk.ID()) } +func waitForStateUpdate(t *testing.T, store *Store, tkID string, firstState *State) { + deadline := time.NewTimer(time.Second) + ticker := time.NewTicker(time.Millisecond) + defer deadline.Stop() + defer ticker.Stop() + for { + state, err := store.Get(tkID) + if err != nil { + t.Fatalf("Get overlapping cancellation state: %v", err) + } + if state.UpdatedAt.After(firstState.UpdatedAt) { + break + } + select { + case <-deadline.C: + t.Fatal("second cancellation did not publish its request") + case <-ticker.C: + } + } +} + func TestCancelOnTerminalTaskIsNoop(t *testing.T) { store := newTestStore(t) task, err := store.Create(CreateOptions{}) @@ -755,11 +910,20 @@ func abandonTask(t *testing.T, store *Store) *State { return state } +func reconcileTask(t *testing.T, store *Store, id string) *State { + t.Helper() + state, err := store.ReconcileTask(context.Background(), id) + if err != nil { + t.Fatalf("ReconcileTask: %v", err) + } + return state +} + func TestReconcileFailsAbandonedTask(t *testing.T) { store := newTestStore(t) state := abandonTask(t, store) - reconciled := store.Reconcile(state) + reconciled := reconcileTask(t, store, state.ID) if reconciled.Status != StatusFailed { t.Errorf("status = %s, want %s", reconciled.Status, StatusFailed) @@ -773,7 +937,7 @@ func TestReconcilePersistsTheFailure(t *testing.T) { store := newTestStore(t) state := abandonTask(t, store) - store.Reconcile(state) + reconcileTask(t, store, state.ID) persisted, err := store.Get(state.ID) if err != nil { @@ -810,7 +974,7 @@ func TestReconcileDoesNotFailARestartedWorker(t *testing.T) { claimed = restarted.HoldWorkerLock() == nil } - reconciled := store.Reconcile(stale) + reconciled := reconcileTask(t, store, stale.ID) if claimed { t.Error("a replacement worker acquired the lock during Reconcile") @@ -846,7 +1010,7 @@ func TestReconcileKeepsAWorkerRecordedResult(t *testing.T) { t.Fatalf("ReleaseWorkerLock: %v", err) } - reconciled := store.Reconcile(stale) + reconciled := reconcileTask(t, store, stale.ID) if reconciled.Status != StatusSucceeded { t.Errorf("status = %s, want %s", reconciled.Status, StatusSucceeded) @@ -868,7 +1032,7 @@ func TestReconcileLeavesLiveTaskAlone(t *testing.T) { t.Fatalf("Get: %v", err) } - if got := store.Reconcile(state).Status; got != StatusPending { + if got := reconcileTask(t, store, state.ID).Status; got != StatusPending { t.Errorf("status = %s, want %s (unchanged)", got, StatusPending) } } @@ -887,7 +1051,7 @@ func TestReconcilePreservesCancellationReason(t *testing.T) { t.Fatalf("Get: %v", err) } - reconciled := store.Reconcile(state) + reconciled := reconcileTask(t, store, state.ID) if reconciled.Error != ErrCanceled.Error() { t.Errorf("error = %q, want %q", reconciled.Error, ErrCanceled.Error()) } @@ -920,22 +1084,62 @@ func setPIDForTest(tk *Task) error { }) } -func setKillProcessForTest(store *Store, fn func(pid, treeName string) error) { - store.killProcessWithIdentity = func(pid, treeName, _ string) error { - return fn(pid, treeName) +type fakeProcessController struct { + terminate func(command.ProcessRef) error + cleanup func(command.ProcessRef) error +} + +func (f fakeProcessController) Terminate(ref command.ProcessRef) error { + if f.terminate == nil { + return nil + } + return f.terminate(ref) +} + +func (f fakeProcessController) CleanupAfterExit(ref command.ProcessRef) error { + if f.cleanup == nil { + return nil + } + return f.cleanup(ref) +} + +func setKillProcessForTest(store *Store, worker *Task, fn func(pid, treeName string) error) { + invoke := func(ref command.ProcessRef) error { + return fn(strconv.Itoa(ref.PID), ref.TreeID) } - store.killExitedWorkerTree = func(pid, treeName, _ string) error { - return fn(pid, treeName) + store.processController = fakeProcessController{ + terminate: func(ref command.ProcessRef) error { + if err := invoke(ref); err != nil { + return err + } + if worker != nil { + _ = worker.ReleaseWorkerLock() + } + return nil + }, + cleanup: invoke, } - store.killLegacyWorkerTree = fn } func setKillProcessWithIdentityForTest( store *Store, + worker *Task, fn func(pid, treeName, identity string) error, ) { - store.killProcessWithIdentity = fn - store.killExitedWorkerTree = fn + store.processController = fakeProcessController{ + terminate: func(ref command.ProcessRef) error { + if err := fn(strconv.Itoa(ref.PID), ref.TreeID, ref.Identity); err != nil { + return err + } + if worker != nil { + _ = worker.ReleaseWorkerLock() + } + return nil + }, + cleanup: func(ref command.ProcessRef) error { + return fn(strconv.Itoa(ref.PID), ref.TreeID, ref.Identity) + }, + } } func requireCanceledState(t *testing.T, store *Store, id string) { @@ -956,7 +1160,7 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { tk := newLiveWorkerTask(t, store) killCalls := 0 - setKillProcessForTest(store, func(pid, treeName string) error { + setKillProcessForTest(store, tk, func(pid, treeName string) error { killCalls++ if killCalls == 1 { return errors.New("boom") @@ -981,8 +1185,8 @@ func TestCancelKillFailureRemainsRetryable(t *testing.T) { if err := tk.Cancel(); err != nil { t.Fatalf("retry Cancel: %v", err) } - if killCalls != 2 { - t.Errorf("kill invoked %d times, want 2 (one per Cancel)", killCalls) + if killCalls != 3 { + t.Errorf("process control invoked %d times, want terminate + retry + cleanup", killCalls) } requireCanceledState(t, store, tk.ID()) } @@ -998,14 +1202,16 @@ func TestCancelLegacyLiveWorkerUsesPIDBoundTreeTermination(t *testing.T) { t.Fatalf("set legacy PID: %v", err) } var actual killCall - store.killLegacyWorkerTree = func(pid, tree string) error { + setKillProcessForTest(store, tk, func(pid, tree string) error { actual = killCall{calls: actual.calls + 1, pid: pid, tree: tree} return nil - } + }) if err := tk.Cancel(); err != nil { t.Fatalf("Cancel: %v", err) } - assertKilledTree(t, actual, "4242", WorkerProcessName(tk.ID())) + if actual.calls != 2 || actual.pid != "4242" || actual.tree != WorkerProcessName(tk.ID()) { + t.Errorf("kill call = %+v, want terminate and cleanup", actual) + } requireCanceledState(t, store, tk.ID()) } @@ -1013,7 +1219,7 @@ func TestCancelSuccessfulKillCommitsCanceled(t *testing.T) { store := newTestStore(t) tk := newLiveWorkerTask(t, store) - setKillProcessForTest(store, func(pid, treeName string) error { + setKillProcessForTest(store, tk, func(pid, treeName string) error { state, err := store.Get(tk.ID()) if err != nil { t.Errorf("Get during kill: %v", err) @@ -1039,19 +1245,24 @@ func TestCancelAwaitsPIDPublication(t *testing.T) { holdWorkerLock(t, tk) var killedPID, killedTree string - setKillProcessForTest(store, func(pid, treeName string) error { + setKillProcessForTest(store, tk, func(pid, treeName string) error { killedPID, killedTree = pid, treeName return nil }) + publishPID := make(chan struct{}) go func() { - time.Sleep(200 * time.Millisecond) + <-publishPID if err := setPIDForTest(tk); err != nil { t.Errorf("setPIDForTest: %v", err) } }() - if err := tk.Cancel(); err != nil { + cancelDone := make(chan error, 1) + go func() { cancelDone <- tk.Cancel() }() + waitForCancelRequest(t, store, tk.ID()) + close(publishPID) + if err := <-cancelDone; err != nil { t.Fatalf("Cancel: %v", err) } if killedPID != "4242" { @@ -1070,13 +1281,18 @@ func TestCancelUnpublishedPIDStaysRetryable(t *testing.T) { t.Fatalf("Create: %v", err) } holdWorkerLock(t, tk) + if err := tk.BeginLaunch(); err != nil { + t.Fatalf("BeginLaunch: %v", err) + } - setKillProcessForTest(store, func(pid, treeName string) error { + setKillProcessForTest(store, tk, func(pid, treeName string) error { t.Error("kill invoked before the worker published a pid") return nil }) - if err := tk.Cancel(); err == nil { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if err := tk.CancelContext(ctx); err == nil { t.Fatal("Cancel = nil, want unpublished-pid error") } state, err := store.Get(tk.ID()) @@ -1105,7 +1321,7 @@ func TestCancelDeadWorkerStillCleansUpProcessTree(t *testing.T) { } var actualKill killCall - setKillProcessForTest(store, func(pid, treeName string) error { + setKillProcessForTest(store, tk, func(pid, treeName string) error { actualKill.calls++ actualKill.pid, actualKill.tree = pid, treeName return nil @@ -1153,7 +1369,7 @@ func TestCancelDeadWorkerCleanupFailureRemainsRetryable(t *testing.T) { if err := setPIDForTest(tk); err != nil { t.Fatalf("setPIDForTest: %v", err) } - setKillProcessForTest(store, func(string, string) error { + setKillProcessForTest(store, tk, func(string, string) error { return errors.New("descendant cleanup failed") }) @@ -1173,7 +1389,7 @@ func TestCancelUsesIdentityCleanupWhenWorkerExitsDuringKill(t *testing.T) { store := newTestStore(t) tk := newLiveWorkerTask(t, store) killCalls := 0 - setKillProcessForTest(store, func(string, string) error { + setKillProcessForTest(store, tk, func(string, string) error { killCalls++ if killCalls == 1 { if err := tk.ReleaseWorkerLock(); err != nil { @@ -1204,7 +1420,7 @@ func TestCancelRacingLateSuccessStillWinsAfterSuccessfulTermination(t *testing.T t.Fatalf("setPIDForTest: %v", err) } - setKillProcessForTest(store, func(pid, treeName string) error { + setKillProcessForTest(store, tk, func(pid, treeName string) error { // The worker reports its own success while the kill unwinds it. return tk.Succeed(&config.Result{}) }) diff --git a/pkg/workspace/delete.go b/pkg/workspace/delete.go index c50ff72784..9563227f93 100644 --- a/pkg/workspace/delete.go +++ b/pkg/workspace/delete.go @@ -26,7 +26,7 @@ type DeleteOptions struct { Force bool ClientDelete client2.DeleteOptions Owner platform.OwnerFilter - quiesceUpTasks func(workspaceID string) error + quiescer UpTaskQuiescer } // Delete deletes a workspace, handling imported workspaces, single-machine @@ -59,7 +59,7 @@ func Delete(ctx context.Context, opts DeleteOptions) (string, error) { // A surviving detached up worker can recreate resources during or after // the delete, so quiescence failure blocks even a forced delete. - if err := opts.quiesce(client.Workspace()); err != nil { + if err := opts.quiesce(ctx, client.Workspace()); err != nil { return "", err } @@ -86,7 +86,7 @@ func Delete(ctx context.Context, opts DeleteOptions) (string, error) { // Re-scan under the lock: a task may have become visible while lock // acquisition waited. - if err := opts.quiesce(client.Workspace()); err != nil { + if err := opts.quiesce(ctx, client.Workspace()); err != nil { return "", err } @@ -245,15 +245,19 @@ func handleDeleteLoadError( return "", loadErr } - return forceDeleteFolder(opts, workspaceID) + return forceDeleteFolder(ctx, opts, workspaceID) } // forceDeleteFolder removes the workspace folder when the workspace client // cannot be loaded and --force is set. -func forceDeleteFolder(opts DeleteOptions, workspaceID string) (string, error) { +func forceDeleteFolder( + ctx context.Context, + opts DeleteOptions, + workspaceID string, +) (string, error) { log.Errorf("error retrieving workspace, force-deleting folder") - if err := opts.quiesce(workspaceID); err != nil { + if err := opts.quiesce(ctx, workspaceID); err != nil { return "", err } @@ -480,12 +484,12 @@ func removeIfContentOrphan(contextName, contentsDir string, entry os.DirEntry) { log.Debugf("removed orphan content dir with no matching workspace: workspace=%s", entry.Name()) } -func (opts DeleteOptions) quiesce(workspaceID string) error { - quiesce := opts.quiesceUpTasks - if quiesce == nil { - quiesce = QuiesceUpTasksForWorkspace +func (opts DeleteOptions) quiesce(ctx context.Context, workspaceID string) error { + quiescer := opts.quiescer + if quiescer == nil { + quiescer = StoreUpTaskQuiescer{} } - if err := quiesce(workspaceID); err != nil { + if err := quiescer.Quiesce(ctx, workspaceID); err != nil { return fmt.Errorf( "cancel detached up tasks for workspace %s: %w", workspaceID, diff --git a/pkg/workspace/delete_test.go b/pkg/workspace/delete_test.go index 79284ad620..06f5b389e1 100644 --- a/pkg/workspace/delete_test.go +++ b/pkg/workspace/delete_test.go @@ -14,6 +14,12 @@ import ( "github.com/stretchr/testify/require" ) +type testQuiescer func(context.Context, string) error + +func (f testQuiescer) Quiesce(ctx context.Context, workspaceID string) error { + return f(ctx, workspaceID) +} + func TestSweepOrphanWorkspaceDirs(t *testing.T) { setupTestPathManager(t) @@ -166,9 +172,9 @@ func TestDeleteBlockedWhenActiveUpTaskCannotBeQuiesced(t *testing.T) { DevsyConfig: devsyConfig, Client: fake, Force: true, - quiesceUpTasks: func(string) error { + quiescer: testQuiescer(func(context.Context, string) error { return errors.New("boom") - }, + }), }) require.Error(t, err) require.False(t, fake.deleted, "delete must not proceed while a live up task survives") diff --git a/pkg/workspace/quiesce.go b/pkg/workspace/quiesce.go index 1739de3f1a..0835d1575d 100644 --- a/pkg/workspace/quiesce.go +++ b/pkg/workspace/quiesce.go @@ -1,8 +1,10 @@ package workspace import ( + "context" "errors" "fmt" + "time" "github.com/devsy-org/devsy/pkg/log" "github.com/devsy-org/devsy/pkg/task" @@ -11,18 +13,25 @@ import ( // upTaskCommand is the command label detached up workers are created with. const upTaskCommand = "up" +const upTaskCancellationTimeout = 5 * time.Second + // QuiesceUpTasks cancels active detached up tasks for the workspace. It tries // every match and returns all failures because a survivor can restart it. func QuiesceUpTasks(store *task.Store, workspaceID string) error { - return quiesceUpTasks(store, workspaceID, func(id string) error { - return store.Open(id).Cancel() + return QuiesceUpTasksContext(context.Background(), store, workspaceID) +} + +func QuiesceUpTasksContext(ctx context.Context, store *task.Store, workspaceID string) error { + return quiesceUpTasks(ctx, store, workspaceID, func(ctx context.Context, id string) error { + return store.Open(id).CancelContext(ctx) }) } func quiesceUpTasks( + ctx context.Context, store *task.Store, workspaceID string, - cancelTask func(id string) error, + cancelTask func(ctx context.Context, id string) error, ) error { active, err := store.ActiveForWorkspace(workspaceID, upTaskCommand) if err != nil { @@ -31,7 +40,10 @@ func quiesceUpTasks( var errs []error for _, state := range active { - if err := cancelTask(state.ID); err != nil { + taskCtx, cancel := context.WithTimeout(ctx, upTaskCancellationTimeout) + err := cancelTask(taskCtx, state.ID) + cancel() + if err != nil { errs = append(errs, fmt.Errorf("cancel up task %s: %w", state.ID, err)) } } @@ -46,12 +58,22 @@ func quiesceUpTasks( return errors.Join(errs...) } -// QuiesceUpTasksForWorkspace quiesces the workspace's active up tasks -// against the default task store. -func QuiesceUpTasksForWorkspace(workspaceID string) error { +type UpTaskQuiescer interface { + Quiesce(context.Context, string) error +} + +type StoreUpTaskQuiescer struct{} + +func (StoreUpTaskQuiescer) Quiesce(ctx context.Context, workspaceID string) error { store, err := task.NewStore() if err != nil { return fmt.Errorf("open task store: %w", err) } - return QuiesceUpTasks(store, workspaceID) + return QuiesceUpTasksContext(ctx, store, workspaceID) +} + +// QuiesceUpTasksForWorkspace quiesces the workspace's active up tasks +// against the default task store. +func QuiesceUpTasksForWorkspace(workspaceID string) error { + return StoreUpTaskQuiescer{}.Quiesce(context.Background(), workspaceID) } diff --git a/pkg/workspace/quiesce_test.go b/pkg/workspace/quiesce_test.go index e036e953bf..dba411d677 100644 --- a/pkg/workspace/quiesce_test.go +++ b/pkg/workspace/quiesce_test.go @@ -1,6 +1,7 @@ package workspace import ( + "context" "errors" "testing" @@ -63,12 +64,17 @@ func TestQuiesceUpTasksAttemptsEveryTaskAndAggregatesFailures(t *testing.T) { tk := tk t.Cleanup(func() { _ = tk.ReleaseWorkerLock() }) } - err := quiesceUpTasks(store, "ws-1", func(id string) error { - if id == failing.ID() { - return errors.New("boom") - } - return store.Open(id).Fail(task.ErrCanceled) - }) + err := quiesceUpTasks( + context.Background(), + store, + "ws-1", + func(_ context.Context, id string) error { + if id == failing.ID() { + return errors.New("boom") + } + return store.Open(id).Fail(task.ErrCanceled) + }, + ) require.Error(t, err) assert.Contains(t, err.Error(), failing.ID()) @@ -80,3 +86,21 @@ func TestQuiesceUpTasksAttemptsEveryTaskAndAggregatesFailures(t *testing.T) { assert.Equal(t, task.StatusFailed, state.Status) assert.Equal(t, task.ErrCanceled.Error(), state.Error) } + +func TestQuiesceUpTasksBoundsEachCancellation(t *testing.T) { + store := newTaskStore(t) + createUpTask(t, store, "ws-1") + + err := quiesceUpTasks( + context.Background(), + store, + "ws-1", + func(ctx context.Context, _ string) error { + if _, ok := ctx.Deadline(); !ok { + return errors.New("task cancellation has no deadline") + } + return nil + }, + ) + require.NoError(t, err) +} From 18429903e0d8e5987bc63aae1d2dd1bd7e6ad58f Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:00:42 -0600 Subject: [PATCH 37/49] fix(task): avoid shadowing builtin copy --- pkg/task/task.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/task/task.go b/pkg/task/task.go index 598f797fca..7e48cc4662 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -514,8 +514,8 @@ func (t *Task) requestCancel() (*State, error) { if !state.Status.Terminal() { state.CancelRequested = true } - copy := *state - snapshot = © + stateSnapshot := *state + snapshot = &stateSnapshot }) if err != nil { return nil, err From 06a0748f031e6565d97137b74520ae79bd7ea769 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:24:50 -0600 Subject: [PATCH 38/49] fix(command): handle process permission errors safely --- pkg/command/process_supported.go | 17 ++++++++++------- pkg/task/task_pid_windows_test.go | 9 +++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index 4c8a67c516..ef2711b8ea 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -28,7 +28,10 @@ func isRunning(pid string) (bool, error) { err = process.Signal(syscall.Signal(0)) if err != nil { - if isExitOrPerm(err) || errors.Is(err, os.ErrProcessDone) { + if errors.Is(err, syscall.EPERM) { + return true, nil + } + if isProcessGone(err) || errors.Is(err, os.ErrProcessDone) { return false, nil } return false, fmt.Errorf("check process %d: %w", parsedPid, err) @@ -82,7 +85,7 @@ func terminateWeakProcessGroup(pid int) error { if err != nil || !running { return err } - if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil && !isExitOrPerm(err) { + if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil && !isProcessGone(err) { return fmt.Errorf("send SIGTERM to worker group %d: %w", pid, err) } return nil @@ -108,7 +111,7 @@ func cleanupExitedProcessRef(ref ProcessRef) error { func abortSupervisedLaunch(pid int, _ string) error { for _, target := range []int{-pid, pid} { if err := syscall.Kill(target, syscall.SIGKILL); err != nil && - !isExitOrPerm(err) { + !isProcessGone(err) { return fmt.Errorf("terminate uncommitted worker %d: %w", pid, err) } } @@ -231,13 +234,13 @@ func checkUnidentifiedProcessGroup(pid int, treeName string) error { ) } -func isExitOrPerm(err error) bool { - return err != nil && (errors.Is(err, syscall.ESRCH) || errors.Is(err, syscall.EPERM)) +func isProcessGone(err error) bool { + return err != nil && errors.Is(err, syscall.ESRCH) } func signalProcessTree(target int, graceful bool, identity string) error { if err := syscall.Kill(target, syscall.SIGTERM); err != nil { - if isExitOrPerm(err) { + if isProcessGone(err) { return nil // already exited } return fmt.Errorf("send SIGTERM to process target %d: %w", target, err) @@ -253,7 +256,7 @@ func signalProcessTree(target int, graceful bool, identity string) error { return nil } err = syscall.Kill(target, syscall.SIGKILL) - if err != nil && !isExitOrPerm(err) { + if err != nil && !isProcessGone(err) { return fmt.Errorf("send SIGKILL to process target %d: %w", target, err) } return nil diff --git a/pkg/task/task_pid_windows_test.go b/pkg/task/task_pid_windows_test.go index 7e8cec6a94..3af1ce7de5 100644 --- a/pkg/task/task_pid_windows_test.go +++ b/pkg/task/task_pid_windows_test.go @@ -3,6 +3,7 @@ package task import ( + "errors" "os/exec" "strconv" "testing" @@ -25,6 +26,14 @@ func TestCancelSignalsALiveWorkersPID(t *testing.T) { if err != nil { t.Fatalf("Create: %v", err) } + holdWorkerLock(t, tk) + controller := command.DefaultProcessController() + store.processController = fakeProcessController{ + terminate: func(ref command.ProcessRef) error { + return errors.Join(controller.Terminate(ref), tk.ReleaseWorkerLock()) + }, + cleanup: controller.CleanupAfterExit, + } worker := sleepWorker() workerName := WorkerProcessName(tk.ID()) if err := command.StartSupervisedBackground(command.SupervisedStartOptions{ From fedbf0be887eadbdcc89414afd61adcc62954034 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:01:20 -0600 Subject: [PATCH 39/49] fix(command): validate saved worker identity --- pkg/command/process_supported.go | 19 +++++++++++++++---- pkg/command/process_windows.go | 15 +++++++++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index ef2711b8ea..2c4eac08c2 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -132,14 +132,25 @@ func killTree(pid, treeName string) error { return fmt.Errorf("read process tree identity for worker %d: %w", parsedPid, err) } if identity == "" { - if treeName == "" { - return nil - } - return checkUnidentifiedProcessGroup(parsedPid, treeName) + return killUnidentifiedTree(parsedPid, treeName) } + return killTreeWithIdentity(pid, treeName, identity) } +func killUnidentifiedTree(parsedPid int, treeName string) error { + if !strongProcessIdentitySupported && treeName == "" { + if err := syscall.Kill(parsedPid, syscall.SIGTERM); err != nil && !isProcessGone(err) { + return fmt.Errorf("send SIGTERM to process %d: %w", parsedPid, err) + } + return nil + } + if treeName == "" { + return nil + } + return checkUnidentifiedProcessGroup(parsedPid, treeName) +} + func killTreeWithIdentity(pid, treeName, identity string) error { parsedPid, err := strconv.Atoi(pid) if err != nil { diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index a027021c01..f03cedc228 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -75,7 +75,7 @@ func terminateProcessRef(ref ProcessRef) error { if terminated { return nil } - running, err := isRunning(strconv.Itoa(ref.PID)) + running, err := savedWorkerRunning(ref) if err != nil { return err } @@ -98,13 +98,24 @@ func terminateProcessRef(ref ProcessRef) error { return fmt.Errorf("unsupported Windows process tree kind %q", ref.TreeKind) } +func savedWorkerRunning(ref ProcessRef) (bool, error) { + current, err := processTreeIdentity(ref.PID) + if err != nil { + return false, err + } + if current == "" || (ref.Identity != "" && current != ref.Identity) { + return false, nil + } + return isRunning(strconv.Itoa(ref.PID)) +} + func cleanupExitedProcessRef(ref ProcessRef) error { if ref.TreeKind == ProcessTreeWindowsJob { terminated, err := terminateNamedJob(ref.TreeID) if err != nil || terminated { return err } - running, err := isRunning(strconv.Itoa(ref.PID)) + running, err := savedWorkerRunning(ref) if err != nil { return err } From ee7eeb25ebf69801ee45452777544c4d2d1c72e0 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:10:25 -0600 Subject: [PATCH 40/49] fix(task): complete abandoned worker cleanup safely --- pkg/command/process_supported.go | 67 +++++++++++-- pkg/command/process_test.go | 53 +++++++++++ pkg/command/process_windows.go | 57 ++++++++++- pkg/command/process_windows_test.go | 29 ++++++ pkg/task/store.go | 57 ++++++++++- pkg/task/store_test.go | 142 ++++++++++++++++++++++++++++ pkg/task/task.go | 54 +++++++---- pkg/workspace/quiesce.go | 37 ++++++-- pkg/workspace/quiesce_test.go | 31 ++++++ 9 files changed, 487 insertions(+), 40 deletions(-) diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index 2c4eac08c2..57d16f194a 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -12,6 +12,12 @@ import ( "time" ) +const ( + weakGroupGracePeriod = 2 * time.Second + weakGroupExitTimeout = 5 * time.Second + weakGroupPollInterval = 100 * time.Millisecond +) + func isRunning(pid string) (bool, error) { parsedPid, err := strconv.Atoi(pid) if err != nil { @@ -81,12 +87,44 @@ func terminateProcessRef(ref ProcessRef) error { } func terminateWeakProcessGroup(pid int) error { - running, err := isRunning(strconv.Itoa(pid)) - if err != nil || !running { + if err := signalWeakProcessGroup(pid, syscall.SIGTERM); err != nil { return err } - if err := syscall.Kill(-pid, syscall.SIGTERM); err != nil && !isProcessGone(err) { - return fmt.Errorf("send SIGTERM to worker group %d: %w", pid, err) + if processGroupExists(pid) { + time.Sleep(weakGroupGracePeriod) + } + if processGroupExists(pid) { + if err := signalWeakProcessGroup(pid, syscall.SIGKILL); err != nil { + return err + } + } + return waitForProcessGroupExit(pid) +} + +func signalWeakProcessGroup(pid int, signal syscall.Signal) error { + if err := syscall.Kill(-pid, signal); err != nil && !isProcessGone(err) { + return fmt.Errorf("send %s to worker group %d: %w", signal, pid, err) + } + return nil +} + +// A worker whose leader exited may still leave members behind. +func processGroupExists(pid int) bool { + err := syscall.Kill(-pid, 0) + if isProcessGone(err) { + return false + } + // EPERM means the group exists but belongs to another user. + return true +} + +func waitForProcessGroupExit(pid int) error { + deadline := time.Now().Add(weakGroupExitTimeout) + for processGroupExists(pid) { + if time.Now().After(deadline) { + return fmt.Errorf("worker group %d still has processes after termination", pid) + } + time.Sleep(weakGroupPollInterval) } return nil } @@ -94,20 +132,37 @@ func terminateWeakProcessGroup(pid int) error { func cleanupExitedProcessRef(ref ProcessRef) error { if ref.TreeKind == ProcessTreeLegacyPID { if ref.Identity == "" { - return nil + return terminateUnidentifiedExitedGroup(ref.PID) } return killTreeAfterWorkerExit(strconv.Itoa(ref.PID), ref.TreeID, ref.Identity) } - if ref.TreeKind != ProcessTreeUnixGroup || ref.Identity == "" { + if ref.TreeKind != ProcessTreeUnixGroup { return nil } groupID := ref.TreeID if groupID == "" { groupID = strconv.Itoa(ref.PID) } + if ref.Identity == "" { + return terminateUnidentifiedExitedGroup(ref.PID) + } return killTreeAfterWorkerExit(groupID, "detached-task", ref.Identity) } +// Never signals the group: after the leader exits the PGID may be reused. +func terminateUnidentifiedExitedGroup(pid int) error { + if pid <= 0 { + return fmt.Errorf("invalid worker PID %d", pid) + } + if processGroupExists(pid) { + return fmt.Errorf( + "worker group %d still has processes and no saved identity identifies them", + pid, + ) + } + return nil +} + func abortSupervisedLaunch(pid int, _ string) error { for _, target := range []int{-pid, pid} { if err := syscall.Kill(target, syscall.SIGKILL); err != nil && diff --git a/pkg/command/process_test.go b/pkg/command/process_test.go index e96fbe5948..9c146563a1 100644 --- a/pkg/command/process_test.go +++ b/pkg/command/process_test.go @@ -267,3 +267,56 @@ func TestKillInvalidPIDReturnsError(t *testing.T) { } } } + +func TestTerminateWeakProcessGroupEndsSigtermIgnoringGroup(t *testing.T) { + cmd := startSigtermIgnoringGroup(t) + pid := cmd.Process.Pid + if err := terminateWeakProcessGroup(pid); err != nil { + t.Fatalf("terminateWeakProcessGroup: %v", err) + } + if processGroupExists(pid) { + t.Errorf("worker group %d still has processes after termination", pid) + } +} + +func TestCleanupExitedIdentitylessGroupReportsSurvivingDescendants(t *testing.T) { + pid := startUnixTestProcess(t, &syscall.SysProcAttr{Setpgid: true}).Process.Pid + ref := ProcessRef{ + PID: pid, + TreeKind: ProcessTreeUnixGroup, + TreeID: strconv.Itoa(pid), + } + + err := cleanupExitedProcessRef(ref) + if err == nil { + t.Fatal("cleanupExitedProcessRef = nil for an identityless group with survivors") + } + if !strings.Contains(err.Error(), "no saved identity") { + t.Fatalf("cleanupExitedProcessRef error = %v, want missing-identity context", err) + } + if running, _ := isRunning(strconv.Itoa(pid)); !running { + t.Fatal("cleanupExitedProcessRef terminated an unidentified group") + } +} + +// startSigtermIgnoringGroup starts a group leader that outlives SIGTERM. The +// child is reaped as soon as it exits, the way a supervising process does, so +// an unreaped zombie does not keep the group looking populated. +func startSigtermIgnoringGroup(t *testing.T) *exec.Cmd { + t.Helper() + cmd := exec.Command("sh", "-c", `trap "" TERM; sleep 30`) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatalf("start SIGTERM-ignoring group: %v", err) + } + reaped := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(reaped) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-reaped + }) + return cmd +} diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index f03cedc228..6476e3c013 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -6,9 +6,12 @@ import ( "errors" "fmt" "os/exec" + "path/filepath" "strconv" + "strings" "time" + "github.com/devsy-org/devsy/pkg/config" "golang.org/x/sys/windows" ) @@ -93,11 +96,63 @@ func terminateProcessRef(ref ProcessRef) error { if ref.Identity != "" { return killTreeWithIdentity(pid, ref.TreeID, ref.Identity) } - return killTree(pid, ref.TreeID) + return terminateLegacyRefWithoutIdentity(ref) } return fmt.Errorf("unsupported Windows process tree kind %q", ref.TreeKind) } +// A record written before process identities existed has nothing to compare a +// reused PID against, so require a Devsy executable before taskkill takes down +// a whole tree. +func terminateLegacyRefWithoutIdentity(ref ProcessRef) error { + image, err := processImageName(ref.PID) + if err != nil { + return err + } + if image == "" { + return nil + } + if !isDevsyImage(image) { + return fmt.Errorf( + "refusing to terminate process %d: %q is not a %s worker", + ref.PID, + filepath.Base(image), + config.RepoName, + ) + } + return killTree(strconv.Itoa(ref.PID), ref.TreeID) +} + +func isDevsyImage(image string) bool { + name := strings.ToLower(filepath.Base(image)) + name = strings.TrimSuffix(name, filepath.Ext(name)) + return name == config.RepoName || strings.HasPrefix(name, config.RepoName+"-") +} + +// processImageName returns an empty string when the process no longer exists. +func processImageName(pid int) (string, error) { + handle, err := windows.OpenProcess( + windows.PROCESS_QUERY_LIMITED_INFORMATION, + false, + uint32(pid), + ) + if err != nil { + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + return "", nil + } + return "", fmt.Errorf("open process %d for image name: %w", pid, err) + } + defer func() { _ = windows.CloseHandle(handle) }() + + const maxPath = 32768 + buffer := make([]uint16, maxPath) + size := uint32(len(buffer)) + if err := windows.QueryFullProcessImageName(handle, 0, &buffer[0], &size); err != nil { + return "", fmt.Errorf("read image name for process %d: %w", pid, err) + } + return windows.UTF16ToString(buffer[:size]), nil +} + func savedWorkerRunning(ref ProcessRef) (bool, error) { current, err := processTreeIdentity(ref.PID) if err != nil { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 4bd591f1cf..52988e8552 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -125,6 +125,35 @@ func assertRunning(t *testing.T, pid int) { } } +func TestIsDevsyImage(t *testing.T) { + for image, want := range map[string]bool{ + `C:\Program Files\Devsy\devsy.exe`: true, + `C:\Program Files\Devsy\devsy.EXE`: true, + `C:\Users\me\bin\devsy-1.19.0-windows-amd64.exe`: true, + `C:\Windows\System32\notepad.exe`: false, + `C:\Users\me\devsync.exe`: false, + "": false, + } { + if got := isDevsyImage(image); got != want { + t.Errorf("isDevsyImage(%q) = %t, want %t", image, got, want) + } + } +} + +func TestTerminateLegacyRefWithoutIdentityRefusesForeignProcess(t *testing.T) { + helper := startHelper(t) + ref := ProcessRef{PID: helper.Process.Pid, TreeKind: ProcessTreeLegacyPID} + + err := terminateLegacyRefWithoutIdentity(ref) + if err == nil { + t.Fatal("terminateLegacyRefWithoutIdentity = nil for a non-Devsy process") + } + if !strings.Contains(err.Error(), "refusing to terminate") { + t.Fatalf("terminateLegacyRefWithoutIdentity error = %v, want a refusal", err) + } + assertRunning(t, helper.Process.Pid) +} + func assertNotRunningEventually(t *testing.T, pid int) { t.Helper() deadline := time.Now().Add(15 * time.Second) diff --git a/pkg/task/store.go b/pkg/task/store.go index 9d63dbc5b9..d44ab5584d 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -153,7 +153,8 @@ func needsReconciliation(state *State) bool { } // ReconcileState marks a task failed after its worker exits without a result. -// It never terminates processes and is safe to use from workspace status. +// It never terminates processes, so the tree is cleaned up by the next +// workspace stop or delete. func (s *Store) ReconcileState(state *State) *State { if !needsStateReconciliation(state) { return state @@ -170,12 +171,39 @@ func (s *Store) ReconcileState(state *State) *State { } return state } - if hasProcessReference(current) { - return current - } return s.failAbandoned(current) } +// CleanupExitedWorkerTree tears down any descendants a dead worker left behind. +// Claiming the worker lock proves no worker is running, so a saved reference +// that still points at live processes belongs to a crashed worker's tree. +func (s *Store) CleanupExitedWorkerTree(id string) error { + state, err := s.Get(id) + if err != nil { + return err + } + if !state.NeedsExitedWorkerCleanup() { + return nil + } + lock, ok := s.claimDeadWorkerLock(id) + if !ok { + return fmt.Errorf("cannot claim dead worker lock for abandoned task %s", id) + } + defer func() { _ = lock.Unlock() }() + state, err = s.Get(id) + if err != nil { + return err + } + if !state.NeedsExitedWorkerCleanup() { + return nil + } + ref, _ := state.ProcessReference() + if err := s.processController.CleanupAfterExit(ref); err != nil { + return fmt.Errorf("clean up task %s process tree: %w", id, err) + } + return s.markProcessCleanupComplete(id) +} + func needsStateReconciliation(state *State) bool { return state != nil && !state.Status.Terminal() && !state.CancelRequested && !state.LaunchPending @@ -273,6 +301,12 @@ func (s *Store) WaitForWorkerObservation( } } +func (s *Store) markProcessCleanupComplete(id string) error { + return s.update(id, func(state *State) { + state.ProcessCleanupComplete = true + }) +} + func (s *Store) workerObservation(id string) (WorkerObservation, bool, error) { state, err := s.Get(id) if err != nil { @@ -297,6 +331,7 @@ func (s *Store) reconcileClaimedTask(ctx context.Context, id string) (*State, er if err != nil || !needsReconciliation(current) { return current, err } + cleaned := false if ref, ok := current.ProcessReference(); ok { if err := ctx.Err(); err != nil { return current, err @@ -304,8 +339,20 @@ func (s *Store) reconcileClaimedTask(ctx context.Context, id string) (*State, er if err := s.processController.CleanupAfterExit(ref); err != nil { return current, fmt.Errorf("clean up task %s process tree: %w", id, err) } + cleaned = true + } + return s.finishClaimedReconciliation(current, cleaned) +} + +func (s *Store) finishClaimedReconciliation(current *State, cleaned bool) (*State, error) { + current = s.failAbandoned(current) + if cleaned && current.Status == StatusFailed && current.Error == ErrAbandoned.Error() { + if err := s.markProcessCleanupComplete(current.ID); err != nil { + return current, err + } + return s.Get(current.ID) } - return s.failAbandoned(current), nil + return current, nil } // failAbandoned records ErrAbandoned for a task whose worker is confirmed gone. diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index d73701ce05..310b4a27ac 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -6,10 +6,13 @@ import ( "time" "github.com/devsy-org/devsy/pkg/command" + "github.com/stretchr/testify/require" ) const workspaceOne = "ws-1" +const testProcessIdentity = "sess-1" + func createTaskWith(t *testing.T, store *Store, opts CreateOptions) *Task { t.Helper() tk, err := store.Create(opts) @@ -28,6 +31,145 @@ func queryActive(t *testing.T, store *Store, workspaceID, command string) []*Sta return active } +func deadWorkerWithProcess( + t *testing.T, + store *Store, + opts CreateOptions, + ref command.ProcessRef, +) *Task { + t.Helper() + tk := createTaskWith(t, store, opts) + if err := tk.SetProcess(ref); err != nil { + t.Fatalf("SetProcess: %v", err) + } + if err := tk.HoldWorkerLock(); err != nil { + t.Fatalf("HoldWorkerLock: %v", err) + } + if err := tk.ReleaseWorkerLock(); err != nil { + t.Fatalf("ReleaseWorkerLock: %v", err) + } + return tk +} + +func TestReconcileStateMarksCrashedWorkerAbandoned(t *testing.T) { + store := newTestStore(t) + tk := deadWorkerWithProcess( + t, + store, + CreateOptions{Command: "up", WorkspaceID: workspaceOne}, + command.ProcessRef{ + PID: 4242, + TreeKind: command.ProcessTreeUnixGroup, + Identity: testProcessIdentity, + }, + ) + + state, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if reconciled := store.ReconcileState(state); !reconciled.Status.Terminal() { + t.Fatalf("ReconcileState status = %q, want terminal", reconciled.Status) + } + + persisted, err := store.Get(tk.ID()) + if err != nil { + t.Fatalf("Get: %v", err) + } + if !persisted.Status.Terminal() { + t.Fatalf("persisted status = %q, want terminal", persisted.Status) + } +} + +func TestCleanupExitedWorkerTreeTerminatesCrashedWorkerTree(t *testing.T) { + store := newTestStore(t) + ref := command.ProcessRef{ + PID: 4242, + TreeKind: command.ProcessTreeUnixGroup, + Identity: testProcessIdentity, + } + tk := deadWorkerWithProcess( + t, + store, + CreateOptions{Command: "up", WorkspaceID: workspaceOne}, + ref, + ) + if err := tk.Fail(ErrAbandoned); err != nil { + t.Fatalf("Fail: %v", err) + } + + var cleaned []command.ProcessRef + store.processController = fakeProcessController{ + cleanup: func(got command.ProcessRef) error { + cleaned = append(cleaned, got) + return nil + }, + } + if err := store.CleanupExitedWorkerTree(tk.ID()); err != nil { + t.Fatalf("CleanupExitedWorkerTree: %v", err) + } + if len(cleaned) != 1 || cleaned[0] != ref { + t.Fatalf("cleaned = %+v, want the saved reference", cleaned) + } + if err := store.CleanupExitedWorkerTree(tk.ID()); err != nil { + t.Fatalf("repeat CleanupExitedWorkerTree: %v", err) + } + if len(cleaned) != 1 { + t.Fatalf("cleaned = %+v after repeat, want one cleanup", cleaned) + } + state, err := store.Get(tk.ID()) + require.NoError(t, err) + require.True(t, state.ProcessCleanupComplete, "successful cleanup was not persisted") +} + +func TestCleanupExitedWorkerTreeSkipsLiveWorker(t *testing.T) { + store := newTestStore(t) + tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) + if err := tk.SetProcess( + command.ProcessRef{ + PID: 4242, + TreeKind: command.ProcessTreeUnixGroup, + Identity: testProcessIdentity, + }, + ); err != nil { + t.Fatalf("SetProcess: %v", err) + } + if err := tk.HoldWorkerLock(); err != nil { + t.Fatalf("HoldWorkerLock: %v", err) + } + t.Cleanup(func() { _ = tk.ReleaseWorkerLock() }) + + store.processController = fakeProcessController{ + cleanup: func(command.ProcessRef) error { + t.Error("cleaned up a live worker's tree") + return nil + }, + } + if err := store.CleanupExitedWorkerTree(tk.ID()); err != nil { + t.Fatalf("CleanupExitedWorkerTree: %v", err) + } +} + +func TestCleanupExitedWorkerTreeDoesNotReportSuccessWhileLockHeld(t *testing.T) { + store := newTestStore(t) + tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) + if err := tk.SetProcess(command.ProcessRef{ + PID: 4242, TreeKind: command.ProcessTreeUnixGroup, Identity: testProcessIdentity, + }); err != nil { + t.Fatalf("SetProcess: %v", err) + } + if err := tk.HoldWorkerLock(); err != nil { + t.Fatalf("HoldWorkerLock: %v", err) + } + t.Cleanup(func() { _ = tk.ReleaseWorkerLock() }) + if err := tk.Fail(ErrAbandoned); err != nil { + t.Fatalf("Fail: %v", err) + } + if err := store.CleanupExitedWorkerTree(tk.ID()); err == nil { + t.Fatal("cleanup reported success while worker lock was held") + } +} + func TestActiveForWorkspaceFilters(t *testing.T) { store := newTestStore(t) diff --git a/pkg/task/task.go b/pkg/task/task.go index 7e48cc4662..b428f731f5 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -46,27 +46,28 @@ func (s Status) Terminal() bool { // doing the work, distinct from whatever process is merely polling this // state. type State struct { - ID string `json:"id"` - Command string `json:"command,omitempty"` - WorkspaceID string `json:"workspaceId,omitempty"` - Status Status `json:"status"` - Phase string `json:"phase,omitempty"` - Step string `json:"step,omitempty"` - OperationID string `json:"operationId,omitempty"` - ParentOperationID string `json:"parentOperationId,omitempty"` - DurationMs int64 `json:"durationMs,omitempty"` - Error string `json:"error,omitempty"` - ErrorCode string `json:"errorCode,omitempty"` - ErrorHint string `json:"errorHint,omitempty"` - ErrorContext map[string]string `json:"errorContext,omitempty"` - Result *config.Result `json:"result,omitempty"` - PID int `json:"pid,omitempty"` - ProcessTreeIdentity string `json:"processTreeIdentity,omitempty"` - Process *command.ProcessRef `json:"process,omitempty"` - CancelRequested bool `json:"cancelRequested,omitempty"` - LaunchPending bool `json:"launchPending,omitempty"` - StartedAt time.Time `json:"startedAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + Command string `json:"command,omitempty"` + WorkspaceID string `json:"workspaceId,omitempty"` + Status Status `json:"status"` + Phase string `json:"phase,omitempty"` + Step string `json:"step,omitempty"` + OperationID string `json:"operationId,omitempty"` + ParentOperationID string `json:"parentOperationId,omitempty"` + DurationMs int64 `json:"durationMs,omitempty"` + Error string `json:"error,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorHint string `json:"errorHint,omitempty"` + ErrorContext map[string]string `json:"errorContext,omitempty"` + Result *config.Result `json:"result,omitempty"` + PID int `json:"pid,omitempty"` + ProcessTreeIdentity string `json:"processTreeIdentity,omitempty"` + Process *command.ProcessRef `json:"process,omitempty"` + ProcessCleanupComplete bool `json:"processCleanupComplete,omitempty"` + CancelRequested bool `json:"cancelRequested,omitempty"` + LaunchPending bool `json:"launchPending,omitempty"` + StartedAt time.Time `json:"startedAt"` + UpdatedAt time.Time `json:"updatedAt"` } // CreateOptions labels a task at creation time for later listing. @@ -112,6 +113,17 @@ func (s *State) ProcessReference() (command.ProcessRef, bool) { }, true } +// NeedsExitedWorkerCleanup identifies abandoned tasks whose saved tree has +// not yet been checked. Other terminal tasks completed their own teardown. +func (s *State) NeedsExitedWorkerCleanup() bool { + if s == nil || s.Status != StatusFailed || s.Error != ErrAbandoned.Error() || + s.ProcessCleanupComplete { + return false + } + _, ok := s.ProcessReference() + return ok +} + func (t *Task) BeginLaunch() error { launcherLock, available, err := t.store.tryLauncherLock(t.id) if err != nil { diff --git a/pkg/workspace/quiesce.go b/pkg/workspace/quiesce.go index 0835d1575d..3547c8f15d 100644 --- a/pkg/workspace/quiesce.go +++ b/pkg/workspace/quiesce.go @@ -33,31 +33,54 @@ func quiesceUpTasks( workspaceID string, cancelTask func(ctx context.Context, id string) error, ) error { - active, err := store.ActiveForWorkspace(workspaceID, upTaskCommand) + states, err := store.ForWorkspace(workspaceID, upTaskCommand) if err != nil { - return fmt.Errorf("list active up tasks for workspace %s: %w", workspaceID, err) + return fmt.Errorf("list up tasks for workspace %s: %w", workspaceID, err) } var errs []error - for _, state := range active { + handled := 0 + for _, state := range states { + if state.Status.Terminal() && !state.NeedsExitedWorkerCleanup() { + continue + } taskCtx, cancel := context.WithTimeout(ctx, upTaskCancellationTimeout) - err := cancelTask(taskCtx, state.ID) + err := quiesceTask(taskCtx, store, state, cancelTask) cancel() if err != nil { - errs = append(errs, fmt.Errorf("cancel up task %s: %w", state.ID, err)) + errs = append(errs, fmt.Errorf("quiesce up task %s: %w", state.ID, err)) + continue } + handled++ } - if len(active) > 0 { + if handled > 0 { log.Debugf( "quiesced up tasks for workspace %s: tasks=%d failures=%d", workspaceID, - len(active), + handled, len(errs), ) } return errors.Join(errs...) } +// A reconciled task is terminal, so cancelling it is a no-op, yet its crashed +// worker may have left descendants that would restart the workspace. +func quiesceTask( + ctx context.Context, + store *task.Store, + state *task.State, + cancelTask func(ctx context.Context, id string) error, +) error { + if !state.Status.Terminal() { + return cancelTask(ctx, state.ID) + } + if err := ctx.Err(); err != nil { + return err + } + return store.CleanupExitedWorkerTree(state.ID) +} + type UpTaskQuiescer interface { Quiesce(context.Context, string) error } diff --git a/pkg/workspace/quiesce_test.go b/pkg/workspace/quiesce_test.go index dba411d677..164acd433c 100644 --- a/pkg/workspace/quiesce_test.go +++ b/pkg/workspace/quiesce_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/task" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,6 +37,9 @@ func TestQuiesceUpTasksCancelsOnlyActiveUpTasksForWorkspace(t *testing.T) { targetA := createUpTask(t, store, "ws-1") targetB := createUpTask(t, store, "ws-1") terminal := createUpTask(t, store, "ws-1") + require.NoError(t, terminal.SetProcess(command.ProcessRef{ + PID: 4242, TreeKind: command.ProcessTreeUnixGroup, TreeID: "4242", + })) require.NoError(t, terminal.Succeed(nil)) otherWorkspace := createUpTask(t, store, "ws-2") otherCommand, err := store.Create(task.CreateOptions{Command: "build", WorkspaceID: "ws-1"}) @@ -49,6 +53,7 @@ func TestQuiesceUpTasksCancelsOnlyActiveUpTasksForWorkspace(t *testing.T) { assert.Equal(t, task.ErrCanceled.Error(), state.Error, "task %s", id) } assert.Equal(t, task.StatusSucceeded, taskState(t, store, terminal.ID()).Status) + assert.False(t, taskState(t, store, terminal.ID()).ProcessCleanupComplete) assert.Equal(t, task.StatusPending, taskState(t, store, otherWorkspace.ID()).Status) assert.Equal(t, task.StatusPending, taskState(t, store, otherCommand.ID()).Status) } @@ -87,6 +92,32 @@ func TestQuiesceUpTasksAttemptsEveryTaskAndAggregatesFailures(t *testing.T) { assert.Equal(t, task.ErrCanceled.Error(), state.Error) } +func TestQuiesceUpTasksCleansTerminalTaskTreeInsteadOfCancelling(t *testing.T) { + store := newTaskStore(t) + crashed := createUpTask(t, store, "ws-1") + require.NoError(t, crashed.SetProcess(command.ProcessRef{ + PID: 4242, + TreeKind: command.ProcessTreeUnixGroup, + TreeID: "4242", + })) + require.NoError(t, crashed.HoldWorkerLock()) + require.NoError(t, crashed.ReleaseWorkerLock()) + require.NoError(t, crashed.Fail(task.ErrAbandoned)) + + var cancelled []string + require.NoError(t, quiesceUpTasks( + context.Background(), + store, + "ws-1", + func(_ context.Context, id string) error { + cancelled = append(cancelled, id) + return nil + }, + )) + + assert.Empty(t, cancelled, "a terminal task must not be cancelled again") +} + func TestQuiesceUpTasksBoundsEachCancellation(t *testing.T) { store := newTaskStore(t) createUpTask(t, store, "ws-1") From 3415f4dd622085cd6f7268a5ac16c7314a7a727d Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:26:32 -0600 Subject: [PATCH 41/49] fix(task): retain legacy worker identity during cleanup --- pkg/command/process_windows.go | 55 +++++++++++++++++++---------- pkg/command/process_windows_test.go | 19 ++++++++++ pkg/task/store.go | 14 +++++--- pkg/task/store_test.go | 21 +++++++++++ 4 files changed, 87 insertions(+), 22 deletions(-) diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index 6476e3c013..c431cccab3 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -70,6 +70,9 @@ func processRefForStartedProcess(pid int, workerName string) (ProcessRef, error) } func terminateProcessRef(ref ProcessRef) error { + if ref.PID <= 0 { + return fmt.Errorf("invalid worker PID %d", ref.PID) + } if ref.TreeKind == ProcessTreeWindowsJob { terminated, err := terminateNamedJob(ref.TreeID) if err != nil { @@ -105,13 +108,34 @@ func terminateProcessRef(ref ProcessRef) error { // reused PID against, so require a Devsy executable before taskkill takes down // a whole tree. func terminateLegacyRefWithoutIdentity(ref ProcessRef) error { - image, err := processImageName(ref.PID) + // Keep the process object open until taskkill returns. Windows cannot reuse + // its PID while this handle exists, even if the worker exits meanwhile. + handle, err := windows.OpenProcess( + windows.PROCESS_QUERY_LIMITED_INFORMATION|windows.SYNCHRONIZE, + false, + uint32(ref.PID), + ) + if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + return nil + } if err != nil { - return err + return fmt.Errorf("open legacy worker %d: %w", ref.PID, err) } - if image == "" { + defer func() { _ = windows.CloseHandle(handle) }() + result, err := windows.WaitForSingleObject(handle, 0) + if err != nil { + return fmt.Errorf("check legacy worker %d: %w", ref.PID, err) + } + if result == windows.WAIT_OBJECT_0 { return nil } + if result != uint32(windows.WAIT_TIMEOUT) { + return fmt.Errorf("check legacy worker %d: unexpected wait result %#x", ref.PID, result) + } + image, err := processImageName(handle, ref.PID) + if err != nil { + return err + } if !isDevsyImage(image) { return fmt.Errorf( "refusing to terminate process %d: %q is not a %s worker", @@ -129,21 +153,7 @@ func isDevsyImage(image string) bool { return name == config.RepoName || strings.HasPrefix(name, config.RepoName+"-") } -// processImageName returns an empty string when the process no longer exists. -func processImageName(pid int) (string, error) { - handle, err := windows.OpenProcess( - windows.PROCESS_QUERY_LIMITED_INFORMATION, - false, - uint32(pid), - ) - if err != nil { - if errors.Is(err, windows.ERROR_INVALID_PARAMETER) { - return "", nil - } - return "", fmt.Errorf("open process %d for image name: %w", pid, err) - } - defer func() { _ = windows.CloseHandle(handle) }() - +func processImageName(handle windows.Handle, pid int) (string, error) { const maxPath = 32768 buffer := make([]uint16, maxPath) size := uint32(len(buffer)) @@ -165,6 +175,9 @@ func savedWorkerRunning(ref ProcessRef) (bool, error) { } func cleanupExitedProcessRef(ref ProcessRef) error { + if ref.PID <= 0 { + return fmt.Errorf("invalid worker PID %d", ref.PID) + } if ref.TreeKind == ProcessTreeWindowsJob { terminated, err := terminateNamedJob(ref.TreeID) if err != nil || terminated { @@ -186,6 +199,12 @@ func cleanupExitedProcessRef(ref ProcessRef) error { if ref.TreeKind == ProcessTreeLegacyPID && ref.Identity != "" { return killTreeAfterWorkerExit(strconv.Itoa(ref.PID), ref.TreeID, ref.Identity) } + if ref.TreeKind == ProcessTreeLegacyPID { + return fmt.Errorf( + "cannot verify descendants of exited legacy worker %d without a saved process identity", + ref.PID, + ) + } return nil } diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 52988e8552..3a0f4c9ae5 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -154,6 +154,25 @@ func TestTerminateLegacyRefWithoutIdentityRefusesForeignProcess(t *testing.T) { assertRunning(t, helper.Process.Pid) } +func TestCleanupExitedLegacyRefWithoutIdentityFailsClosed(t *testing.T) { + err := cleanupExitedProcessRef(ProcessRef{ + PID: 4242, TreeKind: ProcessTreeLegacyPID, + }) + if err == nil || !strings.Contains(err.Error(), "cannot verify descendants") { + t.Fatalf("cleanupExitedProcessRef error = %v, want unverifiable descendants", err) + } +} + +func TestProcessRefRejectsInvalidPID(t *testing.T) { + ref := ProcessRef{TreeKind: ProcessTreeLegacyPID} + if err := terminateProcessRef(ref); err == nil { + t.Fatal("terminateProcessRef accepted PID zero") + } + if err := cleanupExitedProcessRef(ref); err == nil { + t.Fatal("cleanupExitedProcessRef accepted PID zero") + } +} + func assertNotRunningEventually(t *testing.T, pid int) { t.Helper() deadline := time.Now().Add(15 * time.Second) diff --git a/pkg/task/store.go b/pkg/task/store.go index d44ab5584d..02ee008d23 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -72,14 +72,20 @@ func (s *Store) Delete(id string, force bool) error { if err != nil { return err } + if err := s.CleanupExitedWorkerTree(id); err != nil { + return err + } // Locked across the check and the removal so a concurrent update can't // commit its atomic rename after the check and recreate the task. return s.withLock(id, func() error { + state, err := s.Get(id) + if err != nil { + return err + } + if state.NeedsExitedWorkerCleanup() { + return fmt.Errorf("task %s still needs process tree cleanup", id) + } if !force { - state, err := s.Get(id) - if err != nil { - return err - } if !state.Status.Terminal() { return fmt.Errorf( "task %s is still %s; cancel it first or delete with force", diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index 310b4a27ac..0eaa196832 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -170,6 +170,27 @@ func TestCleanupExitedWorkerTreeDoesNotReportSuccessWhileLockHeld(t *testing.T) } } +func TestDeleteKeepsAbandonedTaskWhenTreeCleanupFails(t *testing.T) { + store := newTestStore(t) + tk := deadWorkerWithProcess(t, store, + CreateOptions{Command: "up", WorkspaceID: workspaceOne}, + command.ProcessRef{ + PID: 4242, + TreeKind: command.ProcessTreeUnixGroup, + Identity: testProcessIdentity, + }, + ) + require.NoError(t, tk.Fail(ErrAbandoned)) + store.processController = fakeProcessController{ + cleanup: func(command.ProcessRef) error { return errors.New("cleanup failed") }, + } + for _, force := range []bool{false, true} { + require.ErrorContains(t, store.Delete(tk.ID(), force), "cleanup failed") + _, err := store.Get(tk.ID()) + require.NoError(t, err) + } +} + func TestActiveForWorkspaceFilters(t *testing.T) { store := newTestStore(t) From fab0b003347c0a64a66a27cd8de0234797dd1f71 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Mon, 28 Sep 2026 00:01:11 -0600 Subject: [PATCH 42/49] fix(task): verify terminal worker cleanup across platforms --- pkg/command/process_identity_linux.go | 3 + pkg/command/process_identity_linux_test.go | 12 +++ pkg/command/process_supported.go | 61 +++++++------- pkg/command/process_test.go | 23 +++++- pkg/command/process_windows.go | 17 +++- pkg/command/process_windows_test.go | 43 ++++++++++ pkg/task/store.go | 92 +++++++++++++++++++++- pkg/task/store_test.go | 48 +++++++++++ pkg/task/task.go | 22 +++++- pkg/workspace/quiesce.go | 10 ++- pkg/workspace/quiesce_test.go | 19 +++++ 11 files changed, 305 insertions(+), 45 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index 498c692098..9df583e655 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -201,6 +201,9 @@ func linuxProcessGroupFieldsMatch( return false, true, nil } } + if fields[0] == "Z" || fields[0] == "X" { + return false, false, nil + } return sessionID == wanted.sessionID, false, nil } diff --git a/pkg/command/process_identity_linux_test.go b/pkg/command/process_identity_linux_test.go index e5a65a075f..564f59ba46 100644 --- a/pkg/command/process_identity_linux_test.go +++ b/pkg/command/process_identity_linux_test.go @@ -105,6 +105,18 @@ func TestLinuxProcessGroupMatchTreatsMissingGroupAsGone(t *testing.T) { } } +func TestLinuxProcessGroupFieldsIgnoreExitedMembers(t *testing.T) { + wanted := linuxProcessTreeIdentity{sessionID: 7, startTime: 42} + for _, state := range []string{"Z", "X"} { + member, mismatch, err := linuxProcessGroupFieldsMatch( + 12, 10, wanted, []string{state, "1", "10", "7"}, + ) + if err != nil || member || mismatch { + t.Fatalf("state %s = (%t, %t, %v), want no live member", state, member, mismatch, err) + } + } +} + func TestLinuxProcessGroupProbeHelper(t *testing.T) { if os.Getenv("DEVSY_PROCESS_GROUP_HELPER") != "1" { return diff --git a/pkg/command/process_supported.go b/pkg/command/process_supported.go index 57d16f194a..f5350ed875 100644 --- a/pkg/command/process_supported.go +++ b/pkg/command/process_supported.go @@ -12,12 +12,6 @@ import ( "time" ) -const ( - weakGroupGracePeriod = 2 * time.Second - weakGroupExitTimeout = 5 * time.Second - weakGroupPollInterval = 100 * time.Millisecond -) - func isRunning(pid string) (bool, error) { parsedPid, err := strconv.Atoi(pid) if err != nil { @@ -71,7 +65,7 @@ func terminateProcessRef(ref ProcessRef) error { if ref.Identity != "" { return killTreeWithIdentity(strconv.Itoa(ref.PID), ref.TreeID, ref.Identity) } - return terminateWeakProcessGroup(ref.PID) + return terminateUnidentifiedProcessGroup(ref.PID, strconv.Itoa(ref.PID)) } if ref.TreeKind != ProcessTreeUnixGroup { return fmt.Errorf("unsupported Unix process tree kind %q", ref.TreeKind) @@ -83,29 +77,41 @@ func terminateProcessRef(ref ProcessRef) error { if ref.Identity != "" { return killTreeWithIdentity(groupID, "detached-task", ref.Identity) } - return terminateWeakProcessGroup(ref.PID) + return terminateUnidentifiedProcessGroup(ref.PID, groupID) } -func terminateWeakProcessGroup(pid int) error { - if err := signalWeakProcessGroup(pid, syscall.SIGTERM); err != nil { - return err +func terminateUnidentifiedProcessGroup(pid int, groupID string) error { + if !strongProcessIdentitySupported { + return fmt.Errorf("cannot terminate worker group %d without process identity", pid) } - if processGroupExists(pid) { - time.Sleep(weakGroupGracePeriod) + identity, err := processTreeIdentity(pid) + if err != nil { + return fmt.Errorf("read process identity for worker %d: %w", pid, err) } - if processGroupExists(pid) { - if err := signalWeakProcessGroup(pid, syscall.SIGKILL); err != nil { - return err - } + if identity == "" { + return terminateUnidentifiedExitedGroup(pid) } - return waitForProcessGroupExit(pid) + if err := killTreeWithIdentity(groupID, "detached-task", identity); err != nil { + return err + } + return waitForUnidentifiedGroupExit(pid, identity) } -func signalWeakProcessGroup(pid int, signal syscall.Signal) error { - if err := syscall.Kill(-pid, signal); err != nil && !isProcessGone(err) { - return fmt.Errorf("send %s to worker group %d: %w", signal, pid, err) +func waitForUnidentifiedGroupExit(pid int, identity string) error { + deadline := time.Now().Add(5 * time.Second) + for { + matches, err := processGroupMatchesIdentity(pid, identity) + if err != nil { + return fmt.Errorf("verify worker group %d after termination: %w", pid, err) + } + if !matches { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("worker group %d still has processes after termination", pid) + } + time.Sleep(100 * time.Millisecond) } - return nil } // A worker whose leader exited may still leave members behind. @@ -118,17 +124,6 @@ func processGroupExists(pid int) bool { return true } -func waitForProcessGroupExit(pid int) error { - deadline := time.Now().Add(weakGroupExitTimeout) - for processGroupExists(pid) { - if time.Now().After(deadline) { - return fmt.Errorf("worker group %d still has processes after termination", pid) - } - time.Sleep(weakGroupPollInterval) - } - return nil -} - func cleanupExitedProcessRef(ref ProcessRef) error { if ref.TreeKind == ProcessTreeLegacyPID { if ref.Identity == "" { diff --git a/pkg/command/process_test.go b/pkg/command/process_test.go index 9c146563a1..ba8f145313 100644 --- a/pkg/command/process_test.go +++ b/pkg/command/process_test.go @@ -268,14 +268,29 @@ func TestKillInvalidPIDReturnsError(t *testing.T) { } } -func TestTerminateWeakProcessGroupEndsSigtermIgnoringGroup(t *testing.T) { +func TestTerminateUnidentifiedProcessGroup(t *testing.T) { cmd := startSigtermIgnoringGroup(t) pid := cmd.Process.Pid - if err := terminateWeakProcessGroup(pid); err != nil { - t.Fatalf("terminateWeakProcessGroup: %v", err) + ref := ProcessRef{PID: pid, TreeKind: ProcessTreeLegacyPID} + err := terminateProcessRef(ref) + if !strongProcessIdentitySupported { + if err == nil { + t.Fatal("identityless platform signaled a group it could not verify") + } + if running, _ := isRunning(strconv.Itoa(pid)); !running { + t.Fatal("identityless platform terminated an unidentified worker") + } + return + } + if err != nil { + t.Fatalf("terminateProcessRef: %v", err) + } + deadline := time.Now().Add(5 * time.Second) + for processGroupExists(pid) && time.Now().Before(deadline) { + time.Sleep(50 * time.Millisecond) } if processGroupExists(pid) { - t.Errorf("worker group %d still has processes after termination", pid) + t.Fatalf("worker group %d still has processes after termination", pid) } } diff --git a/pkg/command/process_windows.go b/pkg/command/process_windows.go index c431cccab3..0f5ac8c1af 100644 --- a/pkg/command/process_windows.go +++ b/pkg/command/process_windows.go @@ -144,7 +144,22 @@ func terminateLegacyRefWithoutIdentity(ref ProcessRef) error { config.RepoName, ) } - return killTree(strconv.Itoa(ref.PID), ref.TreeID) + terminated, err := terminateJobForPID(ref.TreeID, ref.PID) + if err != nil { + return err + } + if terminated { + return nil + } + cmd := exec.Command("taskkill", "/PID", strconv.Itoa(ref.PID), "/T", "/F") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf( + "taskkill /PID %d /T /F: %w: %s", + ref.PID, err, truncateOutput(string(output)), + ) + } + return verifyTerminated(ref.PID) } func isDevsyImage(image string) bool { diff --git a/pkg/command/process_windows_test.go b/pkg/command/process_windows_test.go index 3a0f4c9ae5..29ee3f60eb 100644 --- a/pkg/command/process_windows_test.go +++ b/pkg/command/process_windows_test.go @@ -3,6 +3,7 @@ package command import ( + "io" "os" "os/exec" "path/filepath" @@ -154,6 +155,48 @@ func TestTerminateLegacyRefWithoutIdentityRefusesForeignProcess(t *testing.T) { assertRunning(t, helper.Process.Pid) } +func TestTerminateLegacyRefWithoutIdentityEndsDevsyTree(t *testing.T) { + dir := t.TempDir() + t.Setenv(config.EnvHome, dir) + source, err := os.Open(os.Args[0]) // #nosec G304 -- current test binary + if err != nil { + t.Fatalf("open test binary: %v", err) + } + defer func() { _ = source.Close() }() + path := filepath.Join(dir, "devsy.exe") + target, err := os.Create(path) // #nosec G304 -- test-controlled path + if err != nil { + t.Fatalf("create Devsy test binary: %v", err) + } + if _, err := io.Copy(target, source); err != nil { + t.Fatalf("copy Devsy test binary: %v", err) + } + if err := target.Close(); err != nil { + t.Fatalf("close Devsy test binary: %v", err) + } + childPIDFile := filepath.Join(dir, "child.pid") + worker := exec.Command(path, "-test.run", "TestHelperProcess") + worker.Env = helperEnv(helperEnvMarker+"=1", helperEnvPIDFile+"="+childPIDFile) + if err := worker.Start(); err != nil { + t.Fatalf("start legacy Devsy worker: %v", err) + } + t.Cleanup(func() { + _ = worker.Process.Kill() + _ = worker.Wait() + }) + childPID := waitForPIDFile(t, childPIDFile) + ref := ProcessRef{ + PID: worker.Process.Pid, + TreeKind: ProcessTreeLegacyPID, + TreeID: "devsy-up-abcdefghijkl", + } + if err := terminateLegacyRefWithoutIdentity(ref); err != nil { + t.Fatalf("terminate legacy Devsy worker: %v", err) + } + assertNotRunningEventually(t, worker.Process.Pid) + assertNotRunningEventually(t, childPID) +} + func TestCleanupExitedLegacyRefWithoutIdentityFailsClosed(t *testing.T) { err := cleanupExitedProcessRef(ProcessRef{ PID: 4242, TreeKind: ProcessTreeLegacyPID, diff --git a/pkg/task/store.go b/pkg/task/store.go index 02ee008d23..839559790a 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -66,12 +66,18 @@ func (s *Store) Open(id string) *Task { return &Task{store: s, id: id} } -// Delete errors if the task is still pending or running unless force is set. +// Delete cleans any unfinished terminal process tree before removing its +// record. It rejects a non-terminal task unless force is set. func (s *Store) Delete(id string, force bool) error { path, err := s.path(id) if err != nil { return err } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.CleanupCanceledWorkerTree(ctx, id); err != nil { + return err + } if err := s.CleanupExitedWorkerTree(id); err != nil { return err } @@ -82,7 +88,7 @@ func (s *Store) Delete(id string, force bool) error { if err != nil { return err } - if state.NeedsExitedWorkerCleanup() { + if state.NeedsExitedWorkerCleanup() || state.NeedsCanceledWorkerCleanup() { return fmt.Errorf("task %s still needs process tree cleanup", id) } if !force { @@ -210,6 +216,24 @@ func (s *Store) CleanupExitedWorkerTree(id string) error { return s.markProcessCleanupComplete(id) } +// CleanupCanceledWorkerTree handles records that were marked canceled before +// their worker tree was terminated. A successful termination of an identityless +// tree is sufficient only while the worker lock proves it was still live. +func (s *Store) CleanupCanceledWorkerTree(ctx context.Context, id string) error { + if ctx == nil { + ctx = context.Background() + } + state, err := s.Get(id) + if err != nil { + return err + } + if !state.NeedsCanceledWorkerCleanup() { + return nil + } + ref, _ := state.ProcessReference() + return s.waitCanceledWorkerCleanup(ctx, id, ref) +} + func needsStateReconciliation(state *State) bool { return state != nil && !state.Status.Terminal() && !state.CancelRequested && !state.LaunchPending @@ -313,6 +337,70 @@ func (s *Store) markProcessCleanupComplete(id string) error { }) } +func (s *Store) waitCanceledWorkerCleanup( + ctx context.Context, + id string, + ref command.ProcessRef, +) error { + terminated := false + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + if err := ctx.Err(); err != nil { + return err + } + complete, nowTerminated, err := s.cleanupCanceledWorkerTreeStep(id, ref, terminated) + if err != nil || complete { + return err + } + terminated = nowTerminated + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +func (s *Store) cleanupCanceledWorkerTreeStep( + id string, + ref command.ProcessRef, + terminated bool, +) (complete, nowTerminated bool, err error) { + lock, available, err := s.tryWorkerLock(id) + if err != nil { + return false, terminated, fmt.Errorf("inspect canceled task %s worker lock: %w", id, err) + } + if available { + defer func() { _ = lock.Unlock() }() + return true, terminated, s.finishCanceledWorkerCleanup(id, ref, terminated) + } + if !terminated { + if err := s.processController.Terminate(ref); err != nil { + return false, false, fmt.Errorf("terminate canceled task %s process tree: %w", id, err) + } + terminated = true + } + return false, terminated, nil +} + +func (s *Store) finishCanceledWorkerCleanup( + id string, + ref command.ProcessRef, + terminated bool, +) error { + current, err := s.Get(id) + if err != nil || !current.NeedsCanceledWorkerCleanup() { + return err + } + if !terminated || ref.Identity != "" { + if err := s.processController.CleanupAfterExit(ref); err != nil { + return fmt.Errorf("clean up canceled task %s process tree: %w", id, err) + } + } + return s.markProcessCleanupComplete(id) +} + func (s *Store) workerObservation(id string) (WorkerObservation, bool, error) { state, err := s.Get(id) if err != nil { diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index 0eaa196832..9112ec5f06 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -1,6 +1,7 @@ package task import ( + "context" "errors" "testing" "time" @@ -191,6 +192,53 @@ func TestDeleteKeepsAbandonedTaskWhenTreeCleanupFails(t *testing.T) { } } +func TestCleanupCanceledWorkerTreeReapsLegacyLiveWorker(t *testing.T) { + store := newTestStore(t) + tk := createTaskWith(t, store, CreateOptions{Command: "up", WorkspaceID: workspaceOne}) + ref := command.ProcessRef{PID: 4242, TreeKind: command.ProcessTreeLegacyPID} + require.NoError(t, tk.SetProcess(ref)) + require.NoError(t, tk.HoldWorkerLock()) + t.Cleanup(func() { _ = tk.ReleaseWorkerLock() }) + require.NoError(t, tk.Fail(ErrCanceled)) + terminated := 0 + store.processController = fakeProcessController{ + terminate: func(got command.ProcessRef) error { + require.Equal(t, ref, got) + terminated++ + return tk.ReleaseWorkerLock() + }, + cleanup: func(command.ProcessRef) error { + t.Fatal("identityless live tree was cleaned without a verified termination") + return nil + }, + } + require.NoError(t, store.CleanupCanceledWorkerTree(context.Background(), tk.ID())) + require.NoError(t, store.CleanupCanceledWorkerTree(context.Background(), tk.ID())) + require.Equal(t, 1, terminated) + state, err := store.Get(tk.ID()) + require.NoError(t, err) + require.True(t, state.ProcessCleanupComplete) +} + +func TestCleanupCanceledWorkerTreeDoesNotClearUnverifiedLegacyRecord(t *testing.T) { + store := newTestStore(t) + tk := deadWorkerWithProcess(t, store, + CreateOptions{Command: "up", WorkspaceID: workspaceOne}, + command.ProcessRef{PID: 4242, TreeKind: command.ProcessTreeLegacyPID}, + ) + require.NoError(t, tk.Fail(ErrCanceled)) + store.processController = fakeProcessController{ + cleanup: func(command.ProcessRef) error { return errors.New("cannot verify descendants") }, + } + require.ErrorContains(t, + store.CleanupCanceledWorkerTree(context.Background(), tk.ID()), + "cannot verify descendants", + ) + state, err := store.Get(tk.ID()) + require.NoError(t, err) + require.False(t, state.ProcessCleanupComplete) +} + func TestActiveForWorkspaceFilters(t *testing.T) { store := newTestStore(t) diff --git a/pkg/task/task.go b/pkg/task/task.go index b428f731f5..b935927c23 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -124,6 +124,16 @@ func (s *State) NeedsExitedWorkerCleanup() bool { return ok } +// NeedsCanceledWorkerCleanup includes records written by versions that marked +// cancellation complete before the worker tree was actually terminated. +func (s *State) NeedsCanceledWorkerCleanup() bool { + if !s.Canceled() || s.ProcessCleanupComplete { + return false + } + _, ok := s.ProcessReference() + return ok +} + func (t *Task) BeginLaunch() error { launcherLock, available, err := t.store.tryLauncherLock(t.id) if err != nil { @@ -281,10 +291,15 @@ func (t *Task) CancelContext(ctx context.Context) error { if err != nil { return err } - if state.Status.Terminal() { - return nil + if !state.Status.Terminal() { + if err := quiesceCancellation(t, ctx); err != nil { + return err + } + } + if err := t.store.CleanupCanceledWorkerTree(ctx, t.id); err != nil { + return err } - return quiesceCancellation(t, ctx) + return t.store.CleanupExitedWorkerTree(t.id) } func quiesceCancellation(t *Task, ctx context.Context) error { @@ -540,6 +555,7 @@ func (t *Task) finalizeCanceled() error { if !state.Status.Terminal() && state.CancelRequested { markCanceled(state) state.LaunchPending = false + state.ProcessCleanupComplete = true } }) } diff --git a/pkg/workspace/quiesce.go b/pkg/workspace/quiesce.go index 3547c8f15d..7b4ad9a0cf 100644 --- a/pkg/workspace/quiesce.go +++ b/pkg/workspace/quiesce.go @@ -41,7 +41,8 @@ func quiesceUpTasks( var errs []error handled := 0 for _, state := range states { - if state.Status.Terminal() && !state.NeedsExitedWorkerCleanup() { + if state.Status.Terminal() && !state.NeedsExitedWorkerCleanup() && + !state.NeedsCanceledWorkerCleanup() { continue } taskCtx, cancel := context.WithTimeout(ctx, upTaskCancellationTimeout) @@ -73,11 +74,16 @@ func quiesceTask( cancelTask func(ctx context.Context, id string) error, ) error { if !state.Status.Terminal() { - return cancelTask(ctx, state.ID) + if err := cancelTask(ctx, state.ID); err != nil { + return err + } } if err := ctx.Err(); err != nil { return err } + if err := store.CleanupCanceledWorkerTree(ctx, state.ID); err != nil { + return err + } return store.CleanupExitedWorkerTree(state.ID) } diff --git a/pkg/workspace/quiesce_test.go b/pkg/workspace/quiesce_test.go index 164acd433c..0ba7fec789 100644 --- a/pkg/workspace/quiesce_test.go +++ b/pkg/workspace/quiesce_test.go @@ -3,6 +3,7 @@ package workspace import ( "context" "errors" + "runtime" "testing" "github.com/devsy-org/devsy/pkg/command" @@ -118,6 +119,24 @@ func TestQuiesceUpTasksCleansTerminalTaskTreeInsteadOfCancelling(t *testing.T) { assert.Empty(t, cancelled, "a terminal task must not be cancelled again") } +func TestQuiesceUpTasksCleansWorkerThatBecomesAbandonedDuringCancel(t *testing.T) { + store := newTaskStore(t) + tk := createUpTask(t, store, "ws-1") + ref := command.ProcessRef{PID: 99999999, TreeKind: command.ProcessTreeUnixGroup} + if runtime.GOOS == "windows" { + ref.TreeKind = command.ProcessTreeWindowsJob + ref.TreeID = "devsy-test-missing-job" + } + require.NoError(t, tk.SetProcess(ref)) + require.NoError(t, tk.HoldWorkerLock()) + require.NoError(t, tk.ReleaseWorkerLock()) + require.NoError(t, quiesceUpTasks( + context.Background(), store, "ws-1", + func(context.Context, string) error { return tk.Fail(task.ErrAbandoned) }, + )) + assert.True(t, taskState(t, store, tk.ID()).ProcessCleanupComplete) +} + func TestQuiesceUpTasksBoundsEachCancellation(t *testing.T) { store := newTaskStore(t) createUpTask(t, store, "ws-1") From c9211f96bcc1bff4f0d21dd7d4b2ae4dacb3bd30 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:13:34 -0600 Subject: [PATCH 43/49] refactor(command): simplify process group identity check --- pkg/command/process_identity_linux.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index 9df583e655..d081d050ee 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -192,14 +192,9 @@ func linuxProcessGroupFieldsMatch( if err != nil { return false, false, nil } - if pid == pgid { - matches, err := linuxProcessLeaderMatches(pid, sessionID, fields, wanted) - if err != nil { - return false, false, err - } - if !matches { - return false, true, nil - } + leaderMismatch, err := linuxProcessGroupLeaderMismatch(pid, pgid, sessionID, fields, wanted) + if err != nil || leaderMismatch { + return false, leaderMismatch, err } if fields[0] == "Z" || fields[0] == "X" { return false, false, nil @@ -207,6 +202,18 @@ func linuxProcessGroupFieldsMatch( return sessionID == wanted.sessionID, false, nil } +func linuxProcessGroupLeaderMismatch( + pid, pgid, sessionID int, + fields []string, + wanted linuxProcessTreeIdentity, +) (bool, error) { + if pid != pgid { + return false, nil + } + matches, err := linuxProcessLeaderMatches(pid, sessionID, fields, wanted) + return !matches, err +} + func linuxProcessStat(pid int) ([]string, bool, error) { stat, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat")) if err != nil { From ab852e28de52a5766520396577b8fdf4a92761b2 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:28:43 -0600 Subject: [PATCH 44/49] fix(command): resolve process helper collision --- pkg/command/process_identity_linux.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index d081d050ee..8c22829019 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -192,7 +192,13 @@ func linuxProcessGroupFieldsMatch( if err != nil { return false, false, nil } - leaderMismatch, err := linuxProcessGroupLeaderMismatch(pid, pgid, sessionID, fields, wanted) + leaderMismatch, err := linuxProcessGroupFieldsLeaderMismatch( + pid, + pgid, + sessionID, + fields, + wanted, + ) if err != nil || leaderMismatch { return false, leaderMismatch, err } @@ -202,7 +208,7 @@ func linuxProcessGroupFieldsMatch( return sessionID == wanted.sessionID, false, nil } -func linuxProcessGroupLeaderMismatch( +func linuxProcessGroupFieldsLeaderMismatch( pid, pgid, sessionID int, fields []string, wanted linuxProcessTreeIdentity, From f44cf61cd440887afa35a0b125fd836a80f7b45e Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:37:53 -0600 Subject: [PATCH 45/49] fix(command): correct process mismatch assignment --- pkg/command/process_identity_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index 8c22829019..d933e96708 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -192,7 +192,7 @@ func linuxProcessGroupFieldsMatch( if err != nil { return false, false, nil } - leaderMismatch, err := linuxProcessGroupFieldsLeaderMismatch( + leaderMismatch, err = linuxProcessGroupFieldsLeaderMismatch( pid, pgid, sessionID, From 35e7114c8f33062fe7e9d150c1eb95c1ee9f0e5d Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:15:31 -0600 Subject: [PATCH 46/49] fix(command): simplify process leader helper --- pkg/command/process_identity_linux.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index d933e96708..aa7be61073 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -192,13 +192,9 @@ func linuxProcessGroupFieldsMatch( if err != nil { return false, false, nil } - leaderMismatch, err = linuxProcessGroupFieldsLeaderMismatch( - pid, - pgid, - sessionID, - fields, - wanted, - ) + if pid == pgid { + leaderMismatch, err = linuxProcessGroupFieldsLeaderMismatch(pid, sessionID, fields, wanted) + } if err != nil || leaderMismatch { return false, leaderMismatch, err } @@ -209,13 +205,10 @@ func linuxProcessGroupFieldsMatch( } func linuxProcessGroupFieldsLeaderMismatch( - pid, pgid, sessionID int, + pid, sessionID int, fields []string, wanted linuxProcessTreeIdentity, ) (bool, error) { - if pid != pgid { - return false, nil - } matches, err := linuxProcessLeaderMatches(pid, sessionID, fields, wanted) return !matches, err } From 3442fed9721025d0b973f77f5287d47926622443 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:25:32 -0600 Subject: [PATCH 47/49] refactor(command): simplify process group field parsing --- pkg/command/process_identity_linux.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/command/process_identity_linux.go b/pkg/command/process_identity_linux.go index aa7be61073..2ec7e6b6ea 100644 --- a/pkg/command/process_identity_linux.go +++ b/pkg/command/process_identity_linux.go @@ -184,12 +184,8 @@ func linuxProcessGroupFieldsMatch( wanted linuxProcessTreeIdentity, fields []string, ) (member, leaderMismatch bool, err error) { - processGroup, err := strconv.Atoi(fields[2]) - if err != nil || processGroup != pgid { - return false, false, nil - } - sessionID, err := strconv.Atoi(fields[3]) - if err != nil { + processGroup, sessionID, ok := linuxProcessGroupIDs(fields) + if !ok || processGroup != pgid { return false, false, nil } if pid == pgid { @@ -204,6 +200,18 @@ func linuxProcessGroupFieldsMatch( return sessionID == wanted.sessionID, false, nil } +func linuxProcessGroupIDs(fields []string) (processGroup, sessionID int, ok bool) { + if len(fields) < 4 { + return 0, 0, false + } + processGroup, err := strconv.Atoi(fields[2]) + if err != nil { + return 0, 0, false + } + sessionID, err = strconv.Atoi(fields[3]) + return processGroup, sessionID, err == nil +} + func linuxProcessGroupFieldsLeaderMismatch( pid, sessionID int, fields []string, From 5ea848d70833b8d059617f3961f2a8d3de9b0ab6 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:56:04 -0600 Subject: [PATCH 48/49] fix(task): reconcile canceled workers after failed cancellation --- pkg/task/store.go | 41 +++++++++++--- pkg/task/store_test.go | 56 +++++++++++++++++++ .../stop-and-delete-a-workspace.mdx | 2 +- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/pkg/task/store.go b/pkg/task/store.go index 839559790a..3047de90f3 100644 --- a/pkg/task/store.go +++ b/pkg/task/store.go @@ -135,8 +135,8 @@ func (s *Store) Abandoned(state *State) bool { return true } -// ReconcileTask cleans up a task whose worker exited without recording a -// result, then records it as abandoned. +// ReconcileTask cleans up a task whose worker exited before recording its +// result, then records it as canceled or abandoned. func (s *Store) ReconcileTask(ctx context.Context, id string) (*State, error) { if err := ctx.Err(); err != nil { return nil, err @@ -160,13 +160,13 @@ func (s *Store) ReconcileTask(ctx context.Context, id string) (*State, error) { } func needsReconciliation(state *State) bool { - return state != nil && !state.Status.Terminal() && !state.CancelRequested && + return state != nil && !state.Status.Terminal() && !state.LaunchPending } -// ReconcileState marks a task failed after its worker exits without a result. -// It never terminates processes, so the tree is cleaned up by the next -// workspace stop or delete. +// ReconcileState marks a task after its worker exits without a result. A +// requested cancellation completes cleanup here; an abandoned task's tree is +// cleaned up by the next workspace stop or delete. func (s *Store) ReconcileState(state *State) *State { if !needsStateReconciliation(state) { return state @@ -183,6 +183,13 @@ func (s *Store) ReconcileState(state *State) *State { } return state } + if current.CancelRequested { + reconciled, err := s.reconcileClaimedCancellation(context.Background(), current) + if err == nil { + return reconciled + } + return current + } return s.failAbandoned(current) } @@ -235,7 +242,7 @@ func (s *Store) CleanupCanceledWorkerTree(ctx context.Context, id string) error } func needsStateReconciliation(state *State) bool { - return state != nil && !state.Status.Terminal() && !state.CancelRequested && + return state != nil && !state.Status.Terminal() && !state.LaunchPending } @@ -425,6 +432,9 @@ func (s *Store) reconcileClaimedTask(ctx context.Context, id string) (*State, er if err != nil || !needsReconciliation(current) { return current, err } + if current.CancelRequested { + return s.reconcileClaimedCancellation(ctx, current) + } cleaned := false if ref, ok := current.ProcessReference(); ok { if err := ctx.Err(); err != nil { @@ -438,6 +448,23 @@ func (s *Store) reconcileClaimedTask(ctx context.Context, id string) (*State, er return s.finishClaimedReconciliation(current, cleaned) } +func (s *Store) reconcileClaimedCancellation(ctx context.Context, current *State) (*State, error) { + if err := ctx.Err(); err != nil { + return current, err + } + if ref, ok := current.ProcessReference(); ok { + if err := s.processController.CleanupAfterExit(ref); err != nil { + return current, fmt.Errorf( + "clean up canceled task %s process tree: %w", current.ID, err, + ) + } + } + if err := s.Open(current.ID).finalizeCanceled(); err != nil { + return current, err + } + return s.Get(current.ID) +} + func (s *Store) finishClaimedReconciliation(current *State, cleaned bool) (*State, error) { current = s.failAbandoned(current) if cleaned && current.Status == StatusFailed && current.Error == ErrAbandoned.Error() { diff --git a/pkg/task/store_test.go b/pkg/task/store_test.go index 9112ec5f06..208cdcbf46 100644 --- a/pkg/task/store_test.go +++ b/pkg/task/store_test.go @@ -82,6 +82,62 @@ func TestReconcileStateMarksCrashedWorkerAbandoned(t *testing.T) { } } +func TestReconcileTaskFinalizesCanceledWorkerAfterCancellationError(t *testing.T) { + store := newTestStore(t) + ref := command.ProcessRef{ + PID: 4242, TreeKind: command.ProcessTreeUnixGroup, Identity: testProcessIdentity, + } + tk := deadWorkerWithProcess(t, store, + CreateOptions{Command: "up", WorkspaceID: workspaceOne}, ref, + ) + _, err := tk.requestCancel() + require.NoError(t, err) + cleaned := 0 + store.processController = fakeProcessController{ + cleanup: func(got command.ProcessRef) error { + require.Equal(t, ref, got) + cleaned++ + if cleaned == 1 { + return errors.New("cleanup unavailable") + } + return nil + }, + } + _, err = store.ReconcileTask(context.Background(), tk.ID()) + require.ErrorContains(t, err, "cleanup unavailable") + state, err := store.Get(tk.ID()) + require.NoError(t, err) + require.False(t, state.Status.Terminal()) + require.True(t, state.CancelRequested) + + state, err = store.ReconcileTask(context.Background(), tk.ID()) + require.NoError(t, err) + require.True(t, state.Canceled()) + require.True(t, state.ProcessCleanupComplete) + require.Equal(t, 2, cleaned) +} + +func TestReconcileStateFinalizesCanceledWorkerAfterExit(t *testing.T) { + store := newTestStore(t) + ref := command.ProcessRef{ + PID: 4242, TreeKind: command.ProcessTreeUnixGroup, Identity: testProcessIdentity, + } + tk := deadWorkerWithProcess(t, store, + CreateOptions{Command: "up", WorkspaceID: workspaceOne}, ref, + ) + state, err := tk.requestCancel() + require.NoError(t, err) + store.processController = fakeProcessController{ + cleanup: func(got command.ProcessRef) error { + require.Equal(t, ref, got) + return nil + }, + } + reconciled := store.ReconcileState(state) + require.True(t, reconciled.Canceled()) + require.True(t, reconciled.ProcessCleanupComplete) +} + func TestCleanupExitedWorkerTreeTerminatesCrashedWorkerTree(t *testing.T) { store := newTestStore(t) ref := command.ProcessRef{ diff --git a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/stop-and-delete-a-workspace.mdx b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/stop-and-delete-a-workspace.mdx index b362a2605f..339fbce524 100644 --- a/sites/docs-devsy-sh/content/docs/developing-in-workspaces/stop-and-delete-a-workspace.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-in-workspaces/stop-and-delete-a-workspace.mdx @@ -35,7 +35,7 @@ This means you need to start the workspace after it has been stopped when you wa ### Background up tasks Stopping or deleting a workspace first cancels any active detached `devsy workspace up --detach` task for that workspace, so a background up cannot keep recreating or restarting a workspace you asked to stop. -Detached task cancellation is supported on all platforms, including Windows: `devsy workspace task cancel ` terminates the task's process tree and records the task as canceled. +For current detached workers on Linux, macOS, and Windows, `devsy workspace task cancel ` terminates the task's process tree and records the task as canceled. On other Unix systems, or for older task records without a saved process identity, cancellation can fail when Devsy cannot safely identify surviving processes. While an SSH tunnel is attached, its health checks observe the workspace but never start a stopped one. ## Delete a Workspace From 4e5446092687aff07fecdc273472b5a313cbdc84 Mon Sep 17 00:00:00 2001 From: Samuel K <69881238+skevetter@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:51:10 -0600 Subject: [PATCH 49/49] fix(e2e): wait for usable SSH tunnel on Windows --- e2e/framework/command.go | 5 ++++- e2e/framework/retry_test.go | 1 + e2e/tests/ssh/ssh_tunnel_mode.go | 10 ++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 2912067c6c..9e7cb4a274 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -229,10 +229,13 @@ func (f *Framework) DevsySSHOnce( workspace string, command string, ) (string, error) { - out, _, err := f.ExecCommandCapture(ctx, []string{ + out, stderr, err := f.ExecCommandCapture(ctx, []string{ cmdWorkspace, cmdSSH, workspace, flagCommand, command, flagDebug, }) if err != nil { + if stderr != "" { + return "", fmt.Errorf("devsy ssh failed: %w (stderr: %s)", err, stderr) + } return "", fmt.Errorf("devsy ssh failed: %w", err) } return out, nil diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index 79e319b47f..0ccc08961e 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -498,6 +498,7 @@ exit 1 _, err = f.DevsySSHOnce(ctx, "test-ws", "echo test") require.Error(t, err) + assert.Contains(t, err.Error(), "Connection refused") // Read invocation counter data, readErr := os.ReadFile(counterFile) //nolint:gosec // G304 diff --git a/e2e/tests/ssh/ssh_tunnel_mode.go b/e2e/tests/ssh/ssh_tunnel_mode.go index 72232e5f52..3e3f4c7a69 100644 --- a/e2e/tests/ssh/ssh_tunnel_mode.go +++ b/e2e/tests/ssh/ssh_tunnel_mode.go @@ -62,10 +62,12 @@ var _ = ginkgo.Describe( waitDetachedTunnelReady(ctx, f, taskID) - devsySSHCtx, cancelSSH := context.WithDeadline(ctx, time.Now().Add(20*time.Second)) - defer cancelSSH() - err = f.DevsySSHEchoTestString(devsySSHCtx, tempDir) - framework.ExpectNoError(err) + gomega.Eventually(func() (string, error) { + probeCtx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + return f.DevsySSHOnce(probeCtx, tempDir, "echo mYtEsTsTrInG") + }).WithContext(ctx).WithTimeout(tunnelActiveTimeout).WithPolling(2 * time.Second). + Should(gomega.ContainSubstring("mYtEsTsTrInG")) }, )