From 42a453b1dfe857303e53836fd58a5c694e24ae14 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 16:42:32 +0000 Subject: [PATCH 01/14] feat(workspaces): add operation progress and machine diagnostics --- CONTRIBUTING.md | 10 + cmd/internal/agent.go | 1 + cmd/internal/agent_daemon.go | 353 +++++-- cmd/internal/agent_daemon_diagnostics.go | 64 ++ cmd/internal/agent_daemon_test.go | 197 ++++ cmd/internal/agentworkspace/logs_daemon.go | 49 +- cmd/internal/agentworkspace/up.go | 20 +- cmd/internal/logs_daemon.go | 26 +- cmd/machine/diagnostics.go | 233 +++++ cmd/machine/diagnostics_test.go | 133 +++ cmd/machine/logs.go | 92 ++ cmd/machine/machine.go | 2 + cmd/workspace/delete.go | 6 +- cmd/workspace/stop.go | 112 ++- desktop/e2e/fixtures/mock-devsy.cjs | 24 +- desktop/e2e/workspace-progress.e2e.ts | 64 ++ .../src/main/__tests__/ipc-up-tasks.test.ts | 65 +- .../main/__tests__/ipc-workspace-jobs.test.ts | 236 +++-- .../machine-diagnostics-manager.test.ts | 54 + .../machine-diagnostics-store.test.ts | 54 + desktop/src/main/__tests__/tray.test.ts | 87 +- desktop/src/main/__tests__/watcher.test.ts | 39 +- .../src/main/__tests__/workspace-jobs.test.ts | 161 ++- desktop/src/main/index.ts | 31 +- desktop/src/main/ipc.ts | 939 +++++++++--------- .../src/main/machine-diagnostics-manager.ts | 66 ++ desktop/src/main/machine-diagnostics-store.ts | 100 ++ desktop/src/main/tray.ts | 57 +- desktop/src/main/watcher.ts | 63 +- desktop/src/main/workspace-jobs.ts | 165 +-- .../src/lib/components/ErrorCard.svelte | 2 +- .../ui/toggle-group/toggle-group-item.svelte | 2 +- .../ui/toggle-group/toggle-group.svelte | 2 +- .../components/workspace/WorkspaceCard.svelte | 26 +- .../workspace/WorkspaceOperation.svelte | 37 + .../workspace/WorkspaceOperation.test.ts | 58 ++ .../WorkspaceWizard.platform.test.ts | 2 +- .../workspace/WorkspaceWizard.svelte | 18 +- .../workspace/WorkspaceWizard.test.ts | 2 +- desktop/src/renderer/src/lib/ipc/commands.ts | 21 + desktop/src/renderer/src/lib/ipc/events.ts | 8 +- desktop/src/renderer/src/lib/ipc/mock.ts | 9 + .../src/lib/stores/workspaces.test.ts | 243 ++--- .../src/renderer/src/lib/stores/workspaces.ts | 168 ++-- desktop/src/renderer/src/lib/types/index.ts | 20 +- .../src/pages/MachineDetailPage.svelte | 133 ++- .../src/pages/WorkspaceDetailPage.svelte | 94 +- .../renderer/src/pages/WorkspacesPage.svelte | 72 +- .../src/shared/machine-diagnostics-types.ts | 65 ++ desktop/src/shared/workspace-operation.ts | 50 + desktop/vitest.config.ts | 1 + pkg/daemon/agent/daemon.go | 119 ++- pkg/daemon/agent/daemon_test.go | 34 + pkg/daemon/agent/state.go | 97 ++ pkg/daemon/agent/state_test.go | 80 ++ pkg/flags/names/names.go | 60 +- pkg/flags/names/names_test.go | 4 + pkg/machinediagnostics/cursor.go | 42 + pkg/machinediagnostics/locator.go | 85 ++ pkg/machinediagnostics/runtime_lock.go | 43 + pkg/machinediagnostics/sanitize.go | 34 + pkg/machinediagnostics/store.go | 409 ++++++++ pkg/machinediagnostics/store_test.go | 204 ++++ pkg/machinediagnostics/types.go | 159 +++ pkg/status/status.go | 13 + pkg/workspace/delete.go | 85 +- pkg/workspace/delete_progress_test.go | 109 ++ .../stop-and-delete-a-workspace.mdx | 8 +- .../managing-machines/machine-diagnostics.mdx | 102 ++ .../content/docs/managing-machines/meta.json | 2 +- 70 files changed, 4798 insertions(+), 1427 deletions(-) create mode 100644 cmd/internal/agent_daemon_diagnostics.go create mode 100644 cmd/machine/diagnostics.go create mode 100644 cmd/machine/diagnostics_test.go create mode 100644 cmd/machine/logs.go create mode 100644 desktop/e2e/workspace-progress.e2e.ts create mode 100644 desktop/src/main/__tests__/machine-diagnostics-manager.test.ts create mode 100644 desktop/src/main/__tests__/machine-diagnostics-store.test.ts create mode 100644 desktop/src/main/machine-diagnostics-manager.ts create mode 100644 desktop/src/main/machine-diagnostics-store.ts create mode 100644 desktop/src/renderer/src/lib/components/workspace/WorkspaceOperation.svelte create mode 100644 desktop/src/renderer/src/lib/components/workspace/WorkspaceOperation.test.ts create mode 100644 desktop/src/shared/machine-diagnostics-types.ts create mode 100644 desktop/src/shared/workspace-operation.ts create mode 100644 pkg/daemon/agent/state.go create mode 100644 pkg/daemon/agent/state_test.go create mode 100644 pkg/machinediagnostics/cursor.go create mode 100644 pkg/machinediagnostics/locator.go create mode 100644 pkg/machinediagnostics/runtime_lock.go create mode 100644 pkg/machinediagnostics/sanitize.go create mode 100644 pkg/machinediagnostics/store.go create mode 100644 pkg/machinediagnostics/store_test.go create mode 100644 pkg/machinediagnostics/types.go create mode 100644 pkg/workspace/delete_progress_test.go create mode 100644 sites/docs-devsy-sh/content/docs/managing-machines/machine-diagnostics.mdx diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b0586071f..e68c7aaf9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,6 +107,16 @@ task cli:build:dev ### Desktop Development +In renderer code, use `$lib/...` for imports across renderer library modules and +`$shared/...` for shared modules. Apply this to type imports, re-exports, and +dynamic imports as well. Keep `./...` for sibling files and relative paths for +renderer entry points without an existing alias. Avoid parent-directory imports +(`../...`) when one of these aliases addresses the module. + +Main-process, preload, and shared code use relative imports: the renderer aliases +are not configured in the main/preload bundlers. Preserve the existing `.js` +extension on TypeScript module specifiers and `.svelte` on component imports. + ```bash # Install dependencies cd desktop diff --git a/cmd/internal/agent.go b/cmd/internal/agent.go index acbdbf27e..5a6d416c6 100644 --- a/cmd/internal/agent.go +++ b/cmd/internal/agent.go @@ -25,6 +25,7 @@ func NewAgentCmd(globalFlags *flags.GlobalFlags) *cobra.Command { agentCmd.AddCommand(agentworkspace.NewWorkspaceCmd(globalFlags)) agentCmd.AddCommand(agentcontainer.NewContainerCmd(globalFlags)) agentCmd.AddCommand(NewDaemonCmd(globalFlags)) + agentCmd.AddCommand(NewDaemonDiagnosticsCmd(globalFlags)) agentCmd.AddCommand(NewContainerTunnelCmd(globalFlags)) agentCmd.AddCommand(NewGitCredentialsCmd(globalFlags)) agentCmd.AddCommand(NewGitSSHSignatureCmd(globalFlags)) diff --git a/cmd/internal/agent_daemon.go b/cmd/internal/agent_daemon.go index 1f18a205a..f4b2ccefb 100644 --- a/cmd/internal/agent_daemon.go +++ b/cmd/internal/agent_daemon.go @@ -3,7 +3,6 @@ package cmdinternal import ( "bytes" "context" - "errors" "fmt" "os" "path/filepath" @@ -14,11 +13,13 @@ import ( "github.com/devsy-org/devsy/pkg/agent" "github.com/devsy-org/devsy/pkg/client/clientimplementation" agentconfig "github.com/devsy-org/devsy/pkg/config" + agentdaemon "github.com/devsy-org/devsy/pkg/daemon/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" "github.com/devsy-org/devsy/pkg/driver/custom" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/machinediagnostics" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/spf13/cobra" ) @@ -28,11 +29,25 @@ const ( busyGracePeriod = 20 * time.Minute ) +var daemonRuntimeLockPath = machinediagnostics.DefaultRuntimeLockPath + type DaemonCmd struct { *flags.GlobalFlags - Interval string - ShutdownAction string + Interval string + ShutdownAction string + StateRoot string + StateLayout string + DiagnosticsReaderUID int + DiagnosticsReaderGID int + + recorder machinediagnostics.Recorder + startedAt time.Time + lastSuccessfulPatrolAt *time.Time + lastPatrolAt *time.Time + lastDiagnosticError *machinediagnostics.DiagnosticError + knownWorkspaceIDs map[string]struct{} + knownInvalidWorkspaceIDs map[string]struct{} } func NewDaemonCmd(flags *flags.GlobalFlags) *cobra.Command { @@ -54,30 +69,90 @@ func NewDaemonCmd(flags *flags.GlobalFlags) *cobra.Command { "", "The shutdown action (none, stopContainer, or stopCompose)", ), + cliflags.String(&cmd.StateRoot, names.StateRoot, "", "The daemon workspace state root"), + cliflags.String(&cmd.StateLayout, names.StateLayout, "", "The daemon workspace state layout"), + cliflags.Int(&cmd.DiagnosticsReaderUID, names.DiagnosticsReaderUID, -1, "The diagnostics reader UID"), + cliflags.Int(&cmd.DiagnosticsReaderGID, names.DiagnosticsReaderGID, -1, "The diagnostics reader GID"), ) + _ = daemonCmd.Flags().MarkHidden(names.StateRoot) + _ = daemonCmd.Flags().MarkHidden(names.StateLayout) + _ = daemonCmd.Flags().MarkHidden(names.DiagnosticsReaderUID) + _ = daemonCmd.Flags().MarkHidden(names.DiagnosticsReaderGID) return daemonCmd } func (cmd *DaemonCmd) Run(ctx context.Context) error { - // The daemon runs only container/machine-side; the host never invokes it. - if agent.IsHostAgentInvocation(cmd.AgentDir) { - return errors.New( - "`devsy internal agent daemon` is only valid inside the workspace container or machine", - ) + location, err := cmd.stateLocation() + if err != nil { + return err } - - logDir, err := agent.GetAgentDaemonLogDir(cmd.AgentDir) + runtimeLock, err := machinediagnostics.AcquireRuntimeLock(daemonRuntimeLockPath) if err != nil { return err } - - log.Infof("starting Devsy daemon patrol at %s", logDir) + defer func() { _ = runtimeLock.Close() }() + cmd.startedAt = time.Now().UTC() + recorder, err := machinediagnostics.NewRecorder(machinediagnostics.Options{ + Dir: machinediagnostics.DiagnosticsDir(location.Root), + Reader: cmd.diagnosticsReader(), + PatrolInterval: cmd.pollInterval(), + ErrorReporter: func(err error) { + log.Warnf("write machine diagnostics: %v", err) + }, + }) + if err != nil { + log.Warnf("initialize machine diagnostics: %v", err) + cmd.recorder = machinediagnostics.Nop() + } else { + cmd.recorder = recorder + if err := machinediagnostics.WriteLocator(machinediagnostics.DefaultLocatorPath, machinediagnostics.Locator{ + SessionID: recorder.SessionID(), DiagnosticsDir: machinediagnostics.DiagnosticsDir(location.Root), + StateLayout: string(location.Layout), StartedAt: cmd.startedAt, + }); err != nil { + log.Warnf("write machine diagnostics locator: %v", err) + } + } + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventDaemonStarted, Level: machinediagnostics.LevelInfo, Message: "Devsy machine daemon started."}) + cmd.updateDiagnostics(machinediagnostics.DaemonStarting, machinediagnostics.DaemonHealthy, nil, nil, nil) + defer func() { + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventDaemonStopping, Level: machinediagnostics.LevelInfo, Message: "Devsy machine daemon stopping."}) + cmd.updateDiagnostics(machinediagnostics.DaemonStopping, machinediagnostics.DaemonHealthy, nil, nil, nil) + _ = cmd.recorder.Close() + }() + + log.Infof("starting Devsy daemon patrol at %s", location.Root) cmd.patrol(ctx) return nil } +func (cmd *DaemonCmd) stateLocation() (agentdaemon.StateLocation, error) { + if cmd.StateRoot != "" || cmd.StateLayout != "" { + if cmd.StateRoot == "" || cmd.StateLayout == "" { + return agentdaemon.StateLocation{}, fmt.Errorf( + "daemon state root and state layout must be provided together", + ) + } + location := agentdaemon.StateLocation{ + Root: filepath.Clean(cmd.StateRoot), + Layout: agentdaemon.StateLayout(cmd.StateLayout), + } + return location, location.Validate() + } + + if cmd.AgentDir != "" { + location := agentdaemon.StateLocation{ + Root: filepath.Clean(cmd.AgentDir), + Layout: agentdaemon.StateLayoutAgentHome, + } + return location, location.Validate() + } + + return agentdaemon.StateLocation{}, fmt.Errorf("daemon state location is missing") +} + func (cmd *DaemonCmd) patrol(ctx context.Context) { cmd.initialTouch() + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventDaemonReady, Level: machinediagnostics.LevelInfo, Message: "Devsy machine daemon is ready."}) ticker := time.NewTicker(cmd.pollInterval()) defer ticker.Stop() @@ -91,6 +166,36 @@ func (cmd *DaemonCmd) patrol(ctx context.Context) { } } +func (cmd *DaemonCmd) diagnosticsReader() machinediagnostics.ReaderIdentity { + if cmd.DiagnosticsReaderUID >= 0 && cmd.DiagnosticsReaderGID >= 0 { + return machinediagnostics.ReaderIdentity{UID: cmd.DiagnosticsReaderUID, GID: cmd.DiagnosticsReaderGID} + } + return agentdaemon.DiagnosticsReaderIdentity() +} + +func (cmd *DaemonCmd) recordEvent(event machinediagnostics.Event) { + if cmd.recorder != nil { + cmd.recorder.Record(event) + } +} + +func (cmd *DaemonCmd) updateDiagnostics(state machinediagnostics.DaemonState, health machinediagnostics.DaemonHealth, diagnosticErr *machinediagnostics.DiagnosticError, workspaces []machinediagnostics.WorkspaceStatus, candidate *machinediagnostics.ShutdownCandidate) { + if cmd.recorder == nil { + return + } + now := time.Now().UTC() + if diagnosticErr != nil { + cmd.lastDiagnosticError = diagnosticErr + } + if state == machinediagnostics.DaemonRunning { + cmd.lastPatrolAt = &now + } + if health == machinediagnostics.DaemonHealthy && state == machinediagnostics.DaemonRunning { + cmd.lastSuccessfulPatrolAt = &now + } + cmd.recorder.Update(machinediagnostics.Status{StartedAt: cmd.startedAt, UpdatedAt: now, State: state, Health: health, PatrolInterval: cmd.pollInterval().String(), LastPatrolAt: cmd.lastPatrolAt, LastSuccessAt: cmd.lastSuccessfulPatrolAt, LastError: cmd.lastDiagnosticError, WorkspaceCount: len(workspaces), Workspaces: workspaces, ShutdownCandidate: candidate}) +} + func (cmd *DaemonCmd) pollInterval() time.Duration { if cmd.Interval == "" { return defaultPatrolInterval @@ -108,47 +213,66 @@ func (cmd *DaemonCmd) pollInterval() time.Duration { } func (cmd *DaemonCmd) workspaceConfigs() (baseDir string, configs []string, err error) { - baseDir, err = agent.FindAgentHomeDir(cmd.AgentDir) + location, err := cmd.stateLocation() + if err != nil { + return "", nil, err + } + pattern, err := location.WorkspaceConfigPattern() if err != nil { return "", nil, err } - pattern := filepath.Join( - baseDir, - "contexts", - "*", - "workspaces", - "*", - provider2.WorkspaceConfigFile, - ) configs, err = filepath.Glob(pattern) if err != nil { return "", nil, fmt.Errorf("glob %s: %w", pattern, err) } - return baseDir, configs, nil + return location.Root, configs, nil } func (cmd *DaemonCmd) patrolOnce(ctx context.Context) { baseDir, configs, err := cmd.workspaceConfigs() if err != nil { log.Errorf("list workspace configs: %v", err) + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventPatrolFailed, Level: machinediagnostics.LevelError, Message: "Machine daemon could not discover workspace state.", ErrorCode: "patrol_failed"}) + cmd.updateDiagnostics(machinediagnostics.DaemonRunning, machinediagnostics.DaemonDegraded, &machinediagnostics.DiagnosticError{Code: "patrol_failed", Message: "Machine daemon could not discover workspace state.", Timestamp: time.Now().UTC()}, nil, nil) return } - - latestActivity, workspace := findLatestActivity(configs) - if latestActivity == nil { - if len(configs) == 0 { - log.Infof("no workspaces found in %q", baseDir) - } else { - log.Infof( - "%d workspaces found in %q, but none had auto-stop configured or were still running", - len(configs), - baseDir, - ) - } + evaluation := evaluateMachineInactivity(configs, activityHeartbeat(), cmd.ShutdownAction, time.Now()) + cmd.recordWorkspaceTransitions(evaluation.statuses) + cmd.updateDiagnostics(machinediagnostics.DaemonRunning, machinediagnostics.DaemonHealthy, nil, evaluation.statuses, evaluation.candidate) + if evaluation.workspace == nil { + log.Infof("no machine shutdown candidate in %q: %s", baseDir, evaluation.reason) return } + if err := cmd.shutdownWorkspace(ctx, evaluation.workspace); err != nil { + cmd.updateDiagnostics(machinediagnostics.DaemonRunning, machinediagnostics.DaemonDegraded, &machinediagnostics.DiagnosticError{Code: "shutdown_failed", Message: "Machine shutdown action failed.", Timestamp: time.Now().UTC()}, evaluation.statuses, evaluation.candidate) + } +} - cmd.checkAndShutdown(ctx, effectiveActivity(*latestActivity), workspace) +func (cmd *DaemonCmd) recordWorkspaceTransitions(statuses []machinediagnostics.WorkspaceStatus) { + current := make(map[string]struct{}, len(statuses)) + invalid := make(map[string]struct{}) + for _, status := range statuses { + if status.ID == "" { + continue + } + current[status.ID] = struct{}{} + if _, known := cmd.knownWorkspaceIDs[status.ID]; !known { + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventWorkspaceDiscovered, Level: machinediagnostics.LevelInfo, WorkspaceID: status.ID, Message: "Workspace state was discovered."}) + } + if status.State == machinediagnostics.WorkspaceInvalidConfig { + invalid[status.ID] = struct{}{} + if _, alreadyReported := cmd.knownInvalidWorkspaceIDs[status.ID]; !alreadyReported { + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventWorkspaceConfigErr, Level: machinediagnostics.LevelWarn, WorkspaceID: status.ID, Message: "Workspace state could not be parsed.", ErrorCode: "workspace_config_invalid"}) + } + } + } + for id := range cmd.knownWorkspaceIDs { + if _, stillPresent := current[id]; !stillPresent { + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventWorkspaceRemoved, Level: machinediagnostics.LevelInfo, WorkspaceID: id, Message: "Workspace state is no longer present."}) + } + } + cmd.knownWorkspaceIDs = current + cmd.knownInvalidWorkspaceIDs = invalid } var activityFilePath = agentconfig.ContainerActivityFile @@ -168,37 +292,15 @@ func activityHeartbeat() time.Time { return stat.ModTime() } -func (cmd *DaemonCmd) checkAndShutdown( - ctx context.Context, - latestActivity time.Time, - workspace *provider2.AgentWorkspaceInfo, -) { - if cmd.effectiveShutdownAction(workspace) == config.ShutdownActionNone { - return - } - - timeout := agent.DefaultInactivityTimeout - if workspace.Agent.Timeout != "" { - parsed, err := time.ParseDuration(workspace.Agent.Timeout) - if err != nil { - log.Errorf("parse inactivity timeout, using %s: %v", timeout, err) - } else { - timeout = parsed - } - } - - deadline := latestActivity.Add(timeout) - if deadline.After(time.Now()) { - log.Infof( - "workspace %q last active %s, auto-stop in %s", - workspace.Workspace.ID, - latestActivity.Format(time.RFC3339), - time.Until(deadline).Round(time.Second), - ) - return +func (cmd *DaemonCmd) shutdownWorkspace(ctx context.Context, workspace *provider2.AgentWorkspaceInfo) error { + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventIdleDeadline, Level: machinediagnostics.LevelInfo, WorkspaceID: workspace.Workspace.ID, Message: "Workspace inactivity deadline was reached."}) + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventShutdownStarted, Level: machinediagnostics.LevelInfo, WorkspaceID: workspace.Workspace.ID, Message: "Machine shutdown action started."}) + if err := cmd.runShutdownCommand(ctx, workspace); err != nil { + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventShutdownFailed, Level: machinediagnostics.LevelError, WorkspaceID: workspace.Workspace.ID, Message: "Machine shutdown action failed.", ErrorCode: "shutdown_failed"}) + return err } - - cmd.runShutdownCommand(ctx, workspace) + cmd.recordEvent(machinediagnostics.Event{Type: machinediagnostics.EventShutdownSucceeded, Level: machinediagnostics.LevelInfo, WorkspaceID: workspace.Workspace.ID, Message: "Machine shutdown command completed; provider state confirms whether the machine stopped."}) + return nil } // effectiveShutdownAction prefers the workspace's resolved config, falling back @@ -218,11 +320,11 @@ func (cmd *DaemonCmd) effectiveShutdownAction( func (cmd *DaemonCmd) runShutdownCommand( ctx context.Context, workspace *provider2.AgentWorkspaceInfo, -) { +) error { environ, err := custom.ToEnvironWithBinaries(ctx, workspace) if err != nil { log.Errorf("build shutdown environment: %v", err) - return + return err } shutdown := strings.Join(workspace.Agent.Exec.Shutdown, " ") @@ -240,10 +342,11 @@ func (cmd *DaemonCmd) runShutdownCommand( "run shutdown command %s: %v (stdout: %s, stderr: %s)", shutdown, err, stdout.String(), stderr.String(), ) - return + return err } log.Infof("ran shutdown command (stdout: %s, stderr: %s)", stdout.String(), stderr.String()) + return nil } func (cmd *DaemonCmd) initialTouch() { @@ -301,3 +404,117 @@ func getActivity(workspaceConfig string) (*time.Time, *provider2.AgentWorkspaceI } return &activity, workspace, nil } + +// diagnosticsWorkspaceStatuses observes every discovered configuration for the +// status snapshot. It deliberately does not participate in selecting the one +// latest workspace that controls existing shutdown semantics. +type machineInactivityEvaluation struct { + statuses []machinediagnostics.WorkspaceStatus + workspace *provider2.AgentWorkspaceInfo + candidate *machinediagnostics.ShutdownCandidate + reason string +} + +type evaluatedWorkspace struct { + status machinediagnostics.WorkspaceStatus + workspace *provider2.AgentWorkspaceInfo +} + +func evaluateMachineInactivity(configs []string, heartbeat time.Time, fallbackAction string, now time.Time) machineInactivityEvaluation { + evaluated := make([]evaluatedWorkspace, 0, len(configs)) + for _, path := range configs { + evaluated = append(evaluated, evaluateWorkspaceInactivity(path, heartbeat, fallbackAction, now)) + } + result := machineInactivityEvaluation{statuses: make([]machinediagnostics.WorkspaceStatus, 0, len(evaluated))} + for _, item := range evaluated { + result.statuses = append(result.statuses, item.status) + } + for _, item := range evaluated { + if item.status.BlocksMachineShutdown { + result.reason = item.status.BlockerReason + return result + } + } + if len(evaluated) == 0 { + result.reason = "no workspace state was discovered" + return result + } + for _, item := range evaluated { + if result.workspace == nil || item.status.IdleDeadlineAt.After(*result.candidate.EligibleAt) || (item.status.IdleDeadlineAt.Equal(*result.candidate.EligibleAt) && item.status.ID < result.candidate.WorkspaceID) { + deadline := *item.status.IdleDeadlineAt + result.workspace = item.workspace + result.candidate = &machinediagnostics.ShutdownCandidate{WorkspaceID: item.status.ID, EligibleAt: &deadline} + } + } + result.reason = "all workspaces are idle" + return result +} + +func evaluateWorkspaceInactivity(path string, heartbeat time.Time, fallbackAction string, now time.Time) evaluatedWorkspace { + workspace, err := agent.ParseAgentWorkspaceInfo(path) + if err != nil || workspace.Workspace == nil { + return evaluatedWorkspace{status: blockedWorkspaceStatus(workspaceIDFromConfig(path), machinediagnostics.WorkspaceInvalidConfig, "Workspace configuration is invalid")} + } + status := machinediagnostics.WorkspaceStatus{ID: workspace.Workspace.ID} + stat, err := os.Stat(path) + if err != nil { + return evaluatedWorkspace{status: blockedWorkspaceStatus(status.ID, machinediagnostics.WorkspaceNotRunning, "Workspace state is unavailable"), workspace: workspace} + } + action := fallbackAction + if workspace.LastDevContainerConfig != nil && workspace.LastDevContainerConfig.Config != nil && workspace.LastDevContainerConfig.Config.ShutdownAction != "" { + action = workspace.LastDevContainerConfig.Config.ShutdownAction + } + status.ShutdownActionEnabled = len(workspace.Agent.Exec.Shutdown) > 0 && action != config.ShutdownActionNone + if !status.ShutdownActionEnabled { + return evaluatedWorkspace{status: blockedWorkspaceStatus(status.ID, machinediagnostics.WorkspaceNotConfigured, "Auto-stop is not configured"), workspace: workspace} + } + activity := stat.ModTime() + if heartbeat.After(activity) { + activity = heartbeat + } + status.LastActivityAt = &activity + timeout := agent.DefaultInactivityTimeout + if workspace.Agent.Timeout != "" { + parsed, parseErr := time.ParseDuration(workspace.Agent.Timeout) + if parseErr != nil || parsed <= 0 { + return evaluatedWorkspace{status: blockedWorkspaceStatus(status.ID, machinediagnostics.WorkspaceInvalidConfig, "Inactivity timeout must be a positive duration"), workspace: workspace} + } + timeout = parsed + } + status.Timeout = timeout.String() + status.Busy = agent.HasWorkspaceBusyFile(filepath.Dir(path)) + if status.Busy { + deadline := activity.Add(busyGracePeriod).Add(timeout) + status.IdleDeadlineAt = &deadline + status.State = machinediagnostics.WorkspaceBusy + status.BlocksMachineShutdown = true + status.BlockerReason = "Workspace is busy" + return evaluatedWorkspace{status: status, workspace: workspace} + } + deadline := activity.Add(timeout) + status.IdleDeadlineAt = &deadline + if deadline.After(now) { + status.State = machinediagnostics.WorkspaceActive + status.BlocksMachineShutdown = true + status.BlockerReason = "Waiting for inactivity deadline" + } else { + status.State = machinediagnostics.WorkspaceIdleDue + } + return evaluatedWorkspace{status: status, workspace: workspace} +} + +func blockedWorkspaceStatus(id string, state machinediagnostics.WorkspaceEvaluationState, reason string) machinediagnostics.WorkspaceStatus { + return machinediagnostics.WorkspaceStatus{ID: id, State: state, BlocksMachineShutdown: true, BlockerReason: reason} +} + +func diagnosticsWorkspaceStatuses(configs []string) []machinediagnostics.WorkspaceStatus { + return evaluateMachineInactivity(configs, activityHeartbeat(), "", time.Now()).statuses +} + +func workspaceIDFromConfig(path string) string { + parent := filepath.Dir(path) + if filepath.Base(parent) == "agent" { + parent = filepath.Dir(parent) + } + return filepath.Base(parent) +} diff --git a/cmd/internal/agent_daemon_diagnostics.go b/cmd/internal/agent_daemon_diagnostics.go new file mode 100644 index 000000000..296e98b47 --- /dev/null +++ b/cmd/internal/agent_daemon_diagnostics.go @@ -0,0 +1,64 @@ +package cmdinternal + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/devsy-org/devsy/cmd/flags" + agentdaemon "github.com/devsy-org/devsy/pkg/daemon/agent" + cliflags "github.com/devsy-org/devsy/pkg/flags" + "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/devsy-org/devsy/pkg/machinediagnostics" + "github.com/spf13/cobra" +) + +// DaemonDiagnosticsCmd is the machine-side, JSON-only diagnostics reader. +type DaemonDiagnosticsCmd struct { + *flags.GlobalFlags + After string + Limit int + StateRoot string + StateLayout string +} + +func NewDaemonDiagnosticsCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &DaemonDiagnosticsCmd{GlobalFlags: globalFlags} + cobraCmd := &cobra.Command{Use: "daemon-diagnostics", Hidden: true, Args: cobra.NoArgs, RunE: func(c *cobra.Command, _ []string) error { return cmd.Run(c.Context()) }} + cliflags.Add(cobraCmd, + cliflags.String(&cmd.After, "after", "", "An opaque diagnostics cursor"), + cliflags.Int(&cmd.Limit, "limit", machinediagnostics.DefaultReadEvents, "Maximum diagnostic events"), + cliflags.String(&cmd.StateRoot, names.StateRoot, "", "Diagnostics state-root override"), + cliflags.String(&cmd.StateLayout, names.StateLayout, "", "Diagnostics state-layout override"), + ) + _ = cobraCmd.Flags().MarkHidden(names.StateRoot) + _ = cobraCmd.Flags().MarkHidden(names.StateLayout) + return cobraCmd +} + +func (cmd *DaemonDiagnosticsCmd) Run(_ context.Context) error { + if cmd.Limit < 0 || cmd.Limit > machinediagnostics.MaxReadEvents { + return fmt.Errorf("diagnostics limit must be between 0 and %d", machinediagnostics.MaxReadEvents) + } + dir := "" + if cmd.StateRoot != "" || cmd.StateLayout != "" { + if cmd.StateRoot == "" || cmd.StateLayout == "" { + return fmt.Errorf("daemon state root and state layout must be provided together") + } + location := agentdaemon.StateLocation{Root: cmd.StateRoot, Layout: agentdaemon.StateLayout(cmd.StateLayout)} + if err := location.Validate(); err != nil { + return err + } + dir = machinediagnostics.DiagnosticsDir(location.Root) + } else { + response := machinediagnostics.ReadFromLocator(machinediagnostics.DefaultLocatorPath, cmd.After, cmd.Limit, time.Minute, time.Now()) + return json.NewEncoder(os.Stdout).Encode(response) + } + response := machinediagnostics.Read(dir, cmd.After, cmd.Limit, time.Minute, time.Now()) + if dir == "" { + response.Availability = machinediagnostics.AvailabilityNotInitialized + } + return json.NewEncoder(os.Stdout).Encode(response) +} diff --git a/cmd/internal/agent_daemon_test.go b/cmd/internal/agent_daemon_test.go index 75277f0e5..d628c80d7 100644 --- a/cmd/internal/agent_daemon_test.go +++ b/cmd/internal/agent_daemon_test.go @@ -7,8 +7,10 @@ import ( "testing" "time" + "github.com/devsy-org/devsy/cmd/flags" "github.com/devsy-org/devsy/pkg/agent" "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/machinediagnostics" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/types" "github.com/stretchr/testify/assert" @@ -17,6 +19,19 @@ import ( const testEcho = "echo" +type recordingDiagnostics struct { + events []machinediagnostics.Event + statuses []machinediagnostics.Status +} + +func (r *recordingDiagnostics) Record(event machinediagnostics.Event) { + r.events = append(r.events, event) +} +func (r *recordingDiagnostics) Update(status machinediagnostics.Status) { + r.statuses = append(r.statuses, status) +} +func (*recordingDiagnostics) Close() error { return nil } + func writeWorkspaceConfig(t *testing.T, dir string, shutdown types.StrArray) string { t.Helper() require.NoError(t, os.MkdirAll(dir, 0o750)) @@ -155,3 +170,185 @@ func TestEffectiveActivity(t *testing.T) { activityFilePath = touch("stale.activity", time.Now().Add(-2*time.Hour).Truncate(time.Second)) assert.Equal(t, configActivity, effectiveActivity(configActivity)) } + +func TestDaemonStateLocation(t *testing.T) { + t.Run("uses explicit canonical state", func(t *testing.T) { + cmd := &DaemonCmd{StateRoot: "/state", StateLayout: "canonical"} + location, err := cmd.stateLocation() + require.NoError(t, err) + assert.Equal(t, "/state", location.Root) + assert.Equal(t, "canonical", string(location.Layout)) + }) + + t.Run("supports legacy agent directory", func(t *testing.T) { + cmd := &DaemonCmd{GlobalFlags: &flags.GlobalFlags{}} + cmd.AgentDir = "/state/agent" + location, err := cmd.stateLocation() + require.NoError(t, err) + assert.Equal(t, "/state/agent", location.Root) + assert.Equal(t, "agent-home", string(location.Layout)) + }) + + t.Run("rejects incomplete state location", func(t *testing.T) { + _, err := (&DaemonCmd{StateRoot: "/state"}).stateLocation() + require.ErrorContains(t, err, "provided together") + }) +} + +func TestDaemonWorkspaceConfigsSupportsBothLayouts(t *testing.T) { + root := t.TempDir() + canonical := writeWorkspaceConfig( + t, + filepath.Join(root, "contexts", "default", "workspaces", "canonical", "agent"), + types.StrArray{testEcho}, + ) + agentHome := writeWorkspaceConfig( + t, + filepath.Join(root, "contexts", "default", "workspaces", "agent-home"), + types.StrArray{testEcho}, + ) + + t.Run("canonical", func(t *testing.T) { + base, configs, err := (&DaemonCmd{StateRoot: root, StateLayout: "canonical"}).workspaceConfigs() + require.NoError(t, err) + assert.Equal(t, root, base) + assert.Equal(t, []string{canonical}, configs) + }) + + t.Run("agent home", func(t *testing.T) { + base, configs, err := (&DaemonCmd{StateRoot: root, StateLayout: "agent-home"}).workspaceConfigs() + require.NoError(t, err) + assert.Equal(t, root, base) + assert.Equal(t, []string{agentHome}, configs) + }) +} + +func TestDiagnosticsWorkspaceStatuses(t *testing.T) { + root := t.TempDir() + active := writeWorkspaceConfig(t, filepath.Join(root, "contexts", "default", "workspaces", "active", "agent"), types.StrArray{testEcho}) + notConfigured := writeWorkspaceConfig(t, filepath.Join(root, "contexts", "default", "workspaces", "not-configured", "agent"), nil) + invalid := filepath.Join(root, "contexts", "default", "workspaces", "invalid", "agent", provider2.WorkspaceConfigFile) + require.NoError(t, os.MkdirAll(filepath.Dir(invalid), 0o750)) + require.NoError(t, os.WriteFile(invalid, []byte("invalid"), 0o600)) + + statuses := diagnosticsWorkspaceStatuses([]string{active, notConfigured, invalid}) + require.Len(t, statuses, 3) + assert.Equal(t, machinediagnostics.WorkspaceActive, statuses[0].State) + assert.Equal(t, machinediagnostics.WorkspaceNotConfigured, statuses[1].State) + assert.Equal(t, "invalid", statuses[2].ID) + assert.Equal(t, machinediagnostics.WorkspaceInvalidConfig, statuses[2].State) +} + +func TestRecordWorkspaceTransitions(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder} + cmd.recordWorkspaceTransitions([]machinediagnostics.WorkspaceStatus{{ID: "one"}, {ID: "two"}}) + cmd.recordWorkspaceTransitions([]machinediagnostics.WorkspaceStatus{{ID: "two"}, {ID: "three"}}) + require.Len(t, recorder.events, 4) + assert.Equal(t, machinediagnostics.EventWorkspaceDiscovered, recorder.events[0].Type) + assert.Equal(t, "one", recorder.events[0].WorkspaceID) + assert.Equal(t, machinediagnostics.EventWorkspaceDiscovered, recorder.events[2].Type) + assert.Equal(t, "three", recorder.events[2].WorkspaceID) + assert.Equal(t, machinediagnostics.EventWorkspaceRemoved, recorder.events[3].Type) + assert.Equal(t, "one", recorder.events[3].WorkspaceID) +} + +func TestRecordWorkspaceTransitionsReportsInvalidConfigOnce(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder} + invalid := []machinediagnostics.WorkspaceStatus{{ID: "broken", State: machinediagnostics.WorkspaceInvalidConfig}} + cmd.recordWorkspaceTransitions(invalid) + cmd.recordWorkspaceTransitions(invalid) + require.Len(t, recorder.events, 2) + assert.Equal(t, machinediagnostics.EventWorkspaceDiscovered, recorder.events[0].Type) + assert.Equal(t, machinediagnostics.EventWorkspaceConfigErr, recorder.events[1].Type) +} + +func TestUpdateDiagnosticsPreservesLastSuccessfulPatrolAfterFailure(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder, startedAt: time.Now().UTC(), Interval: "1m"} + cmd.updateDiagnostics(machinediagnostics.DaemonRunning, machinediagnostics.DaemonHealthy, nil, nil, nil) + cmd.updateDiagnostics(machinediagnostics.DaemonRunning, machinediagnostics.DaemonDegraded, &machinediagnostics.DiagnosticError{Code: "patrol_failed"}, nil, nil) + require.Len(t, recorder.statuses, 2) + require.NotNil(t, recorder.statuses[0].LastSuccessAt) + require.NotNil(t, recorder.statuses[1].LastSuccessAt) + assert.Equal(t, *recorder.statuses[0].LastSuccessAt, *recorder.statuses[1].LastSuccessAt) +} + +func TestDiagnosticsRetainsErrorHistoryWithoutInventingStartupPatrol(t *testing.T) { + recorder := &recordingDiagnostics{} + cmd := &DaemonCmd{recorder: recorder} + cmd.updateDiagnostics(machinediagnostics.DaemonStarting, machinediagnostics.DaemonHealthy, nil, nil, nil) + assert.Nil(t, recorder.statuses[0].LastPatrolAt) + cmd.updateDiagnostics(machinediagnostics.DaemonRunning, machinediagnostics.DaemonDegraded, &machinediagnostics.DiagnosticError{Code: "shutdown_failed"}, nil, nil) + cmd.updateDiagnostics(machinediagnostics.DaemonRunning, machinediagnostics.DaemonHealthy, nil, nil, nil) + assert.Equal(t, "shutdown_failed", recorder.statuses[2].LastError.Code) + assert.Equal(t, machinediagnostics.DaemonHealthy, recorder.statuses[2].Health) +} + +func TestInvalidTimeoutBlocksShutdown(t *testing.T) { + for _, timeout := range []string{"invalid", "0s", "-1m"} { + t.Run(timeout, func(t *testing.T) { + path := writeWorkspaceConfig(t, t.TempDir(), types.StrArray{testEcho}) + info, err := agent.ParseAgentWorkspaceInfo(path) + require.NoError(t, err) + info.Agent.Timeout = timeout + data, err := json.Marshal(info) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, data, 0o600)) + evaluation := evaluateMachineInactivity([]string{path}, time.Time{}, config.ShutdownActionStopContainer, time.Now().Add(24*time.Hour)) + assert.Nil(t, evaluation.candidate) + require.Len(t, evaluation.statuses, 1) + assert.Equal(t, machinediagnostics.WorkspaceInvalidConfig, evaluation.statuses[0].State) + assert.True(t, evaluation.statuses[0].BlocksMachineShutdown) + }) + } +} + +func TestEvaluateMachineInactivityRequiresAllWorkspacesToBeIdle(t *testing.T) { + root := t.TempDir() + first := writeWorkspaceConfig(t, filepath.Join(root, "one"), types.StrArray{testEcho}) + second := writeWorkspaceConfig(t, filepath.Join(root, "two"), types.StrArray{testEcho}) + now := time.Now().UTC().Truncate(time.Second) + old := now.Add(-2 * agent.DefaultInactivityTimeout) + require.NoError(t, os.Chtimes(first, old, old)) + require.NoError(t, os.Chtimes(second, old, old)) + + evaluation := evaluateMachineInactivity([]string{first, second}, time.Time{}, config.ShutdownActionStopContainer, now) + require.NotNil(t, evaluation.workspace) + require.NotNil(t, evaluation.candidate) + assert.Equal(t, "all workspaces are idle", evaluation.reason) + + require.NoError(t, os.Chtimes(second, now, now)) + evaluation = evaluateMachineInactivity([]string{first, second}, time.Time{}, config.ShutdownActionStopContainer, now) + assert.Nil(t, evaluation.workspace) + assert.Equal(t, "Waiting for inactivity deadline", evaluation.reason) + assert.Len(t, evaluation.statuses, 2) +} + +func TestEvaluateMachineInactivityUsesHeartbeatAndBusyGrace(t *testing.T) { + configPath := writeWorkspaceConfig(t, t.TempDir(), types.StrArray{testEcho}) + now := time.Now().UTC().Truncate(time.Second) + old := now.Add(-2 * agent.DefaultInactivityTimeout) + require.NoError(t, os.Chtimes(configPath, old, old)) + heartbeat := now.Add(-time.Minute) + evaluation := evaluateMachineInactivity([]string{configPath}, heartbeat, config.ShutdownActionStopContainer, now) + require.Len(t, evaluation.statuses, 1) + assert.Equal(t, machinediagnostics.WorkspaceActive, evaluation.statuses[0].State) + assert.Equal(t, heartbeat, *evaluation.statuses[0].LastActivityAt) + + agent.CreateWorkspaceBusyFile(filepath.Dir(configPath)) + evaluation = evaluateMachineInactivity([]string{configPath}, time.Time{}, config.ShutdownActionStopContainer, now) + assert.Equal(t, machinediagnostics.WorkspaceBusy, evaluation.statuses[0].State) + require.NotNil(t, evaluation.statuses[0].IdleDeadlineAt) + assert.WithinDuration(t, old.Add(busyGracePeriod).Add(agent.DefaultInactivityTimeout), *evaluation.statuses[0].IdleDeadlineAt, time.Second) +} + +func TestEvaluateMachineInactivityBlocksDisabledWorkspace(t *testing.T) { + configPath := writeWorkspaceConfig(t, t.TempDir(), types.StrArray{testEcho}) + evaluation := evaluateMachineInactivity([]string{configPath}, time.Time{}, config.ShutdownActionNone, time.Now()) + assert.Nil(t, evaluation.workspace) + require.Len(t, evaluation.statuses, 1) + assert.True(t, evaluation.statuses[0].BlocksMachineShutdown) + assert.Equal(t, "Auto-stop is not configured", evaluation.statuses[0].BlockerReason) +} diff --git a/cmd/internal/agentworkspace/logs_daemon.go b/cmd/internal/agentworkspace/logs_daemon.go index 2d757b566..4d7d4d37e 100644 --- a/cmd/internal/agentworkspace/logs_daemon.go +++ b/cmd/internal/agentworkspace/logs_daemon.go @@ -2,15 +2,14 @@ package agentworkspace import ( "context" - "fmt" - "io" + "encoding/json" "os" - "path/filepath" + "time" "github.com/devsy-org/devsy/cmd/flags" - "github.com/devsy-org/devsy/pkg/agent" cliflags "github.com/devsy-org/devsy/pkg/flags" "github.com/devsy-org/devsy/pkg/flags/names" + "github.com/devsy-org/devsy/pkg/machinediagnostics" "github.com/spf13/cobra" ) @@ -40,39 +39,13 @@ func NewLogsDaemonCmd(flags *flags.GlobalFlags) *cobra.Command { } func (cmd *LogsDaemonCmd) Run(ctx context.Context) error { - // `agent workspace logs-daemon` reads agent-daemon.log, which only - // exists inside the workspace container or machine. Reject host - // invocations explicitly to surface misconfigurations early. - if agent.IsHostAgentInvocation(cmd.AgentDir) { - return fmt.Errorf( - "`devsy internal agent workspace logs-daemon` is only valid inside the workspace container or machine", - ) - } - - // get workspace - shouldExit, _, err := agent.ReadAgentWorkspaceInfo( - cmd.AgentDir, - cmd.Context, - cmd.ID, + _ = ctx + response := machinediagnostics.ReadFromLocator( + machinediagnostics.DefaultLocatorPath, + "", + machinediagnostics.DefaultReadEvents, + 0, + time.Now(), ) - if err != nil { - return err - } else if shouldExit { - return nil - } - - logDir, err := agent.GetAgentDaemonLogDir(cmd.AgentDir) - if err != nil { - return err - } - - // #nosec G304 -- reads the agent's own daemon log at a derived path. - f, err := os.Open(filepath.Join(logDir, "agent-daemon.log")) - if err != nil { - return fmt.Errorf("open agent-daemon.log: %w", err) - } - defer func() { _ = f.Close() }() - - _, err = io.Copy(os.Stdout, f) - return err + return json.NewEncoder(os.Stdout).Encode(response) } diff --git a/cmd/internal/agentworkspace/up.go b/cmd/internal/agentworkspace/up.go index 204cbb0f6..24fef5990 100644 --- a/cmd/internal/agentworkspace/up.go +++ b/cmd/internal/agentworkspace/up.go @@ -767,11 +767,21 @@ func installDaemon(workspaceInfo *provider.AgentWorkspaceInfo) error { } log.Debugf("installing Devsy daemon into server") - return agentdaemon.InstallDaemon( - workspaceInfo.Agent.DataPath, - workspaceInfo.CLIOptions.DaemonInterval, - shutdownAction, - ) + location, err := agentdaemon.ResolveStateLocation(agentdaemon.ResolveStateLocationOptions{ + AgentDataPath: workspaceInfo.Agent.DataPath, + Origin: workspaceInfo.Origin, + Context: workspaceInfo.Workspace.Context, + WorkspaceID: workspaceInfo.Workspace.ID, + }) + if err != nil { + return fmt.Errorf("resolve daemon state location: %w", err) + } + return agentdaemon.InstallDaemon(agentdaemon.InstallOptions{ + StateLocation: location, + Interval: workspaceInfo.CLIOptions.DaemonInterval, + ShutdownAction: shutdownAction, + DiagnosticsReader: agentdaemon.DiagnosticsReaderIdentity(), + }) } func persistResolvedConfig( diff --git a/cmd/internal/logs_daemon.go b/cmd/internal/logs_daemon.go index d03ad3efe..892a410d1 100644 --- a/cmd/internal/logs_daemon.go +++ b/cmd/internal/logs_daemon.go @@ -1,13 +1,17 @@ package cmdinternal import ( + "bytes" "context" + "encoding/json" "fmt" "os" + "time" "github.com/devsy-org/devsy/cmd/flags" "github.com/devsy-org/devsy/pkg/client" "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/machinediagnostics" provider2 "github.com/devsy-org/devsy/pkg/provider" "github.com/devsy-org/devsy/pkg/workspace" "github.com/spf13/cobra" @@ -74,10 +78,24 @@ func (cmd *LogsDaemonCmd) Run(ctx context.Context, args []string) error { command += fmt.Sprintf(" --agent-dir %q", agentInfo.Agent.DataPath) } - // read daemon logs - return workspaceClient.Command(ctx, client.CommandOptions{ + var stdout bytes.Buffer + if err := workspaceClient.Command(ctx, client.CommandOptions{ Command: command, - Stdout: os.Stdout, + Stdout: &stdout, Stderr: os.Stderr, - }) + }); err != nil { + return err + } + + var response machinediagnostics.ReadResponse + if err := json.Unmarshal(stdout.Bytes(), &response); err != nil { + return fmt.Errorf("decode remote daemon diagnostics: %w", err) + } + for _, event := range response.Events { + _, _ = fmt.Fprintf(os.Stdout, "%s %-5s %-35s %s\n", event.Timestamp.Format(time.RFC3339), event.Level, event.Type, event.Message) + } + if response.Availability != machinediagnostics.AvailabilityAvailable { + _, _ = fmt.Fprintf(os.Stdout, "Daemon diagnostics: %s\n", response.Availability) + } + return nil } diff --git a/cmd/machine/diagnostics.go b/cmd/machine/diagnostics.go new file mode 100644 index 000000000..72cbc71fa --- /dev/null +++ b/cmd/machine/diagnostics.go @@ -0,0 +1,233 @@ +package machine + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strconv" + "strings" + "text/tabwriter" + "time" + + "al.essio.dev/pkg/shellescape" + "github.com/devsy-org/devsy/cmd/flags" + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/config" + cliflags "github.com/devsy-org/devsy/pkg/flags" + "github.com/devsy-org/devsy/pkg/machinediagnostics" + "github.com/devsy-org/devsy/pkg/output" + "github.com/devsy-org/devsy/pkg/workspace" + "github.com/spf13/cobra" +) + +const maxRemoteDiagnosticsResponse = 2 * 1024 * 1024 + +type boundedDiagnosticsBuffer struct { + bytes.Buffer + max int +} + +func (b *boundedDiagnosticsBuffer) Write(value []byte) (int, error) { + if b.Len()+len(value) > b.max { + return 0, fmt.Errorf("remote diagnostics response exceeds %d bytes", b.max) + } + return b.Buffer.Write(value) +} + +type CollectionSource struct { + Availability string `json:"availability"` + Freshness string `json:"freshness"` + ErrorCode string `json:"errorCode,omitempty"` + Message string `json:"message,omitempty"` +} +type MachineDiagnostics struct { + SchemaVersion int `json:"schemaVersion"` + Machine struct { + ID string `json:"id"` + Context string `json:"context"` + Provider string `json:"provider"` + State string `json:"state"` + } `json:"machine"` + Source CollectionSource `json:"source"` + Daemon *machinediagnostics.Status `json:"daemon,omitempty"` + Events []machinediagnostics.Event `json:"events,omitempty"` + Cursor machinediagnostics.CursorInfo `json:"cursor"` +} + +type DiagnosticsCmd struct { + *flags.GlobalFlags + After string + Limit int + NoEvents bool +} + +func NewDiagnosticsCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &DiagnosticsCmd{GlobalFlags: globalFlags} + c := &cobra.Command{Use: "diagnostics [name]", Short: "Show Devsy machine diagnostic events and daemon status", RunE: func(c *cobra.Command, args []string) error { return cmd.Run(c.Context(), args) }} + cliflags.Add(c, cliflags.String(&cmd.After, "after", "", "An opaque diagnostics cursor"), cliflags.Int(&cmd.Limit, "limit", 20, "Maximum diagnostic events"), cliflags.Bool(&cmd.NoEvents, "no-events", false, "Do not include diagnostic events")) + return c +} +func (cmd *DiagnosticsCmd) Run(ctx context.Context, args []string) error { + if cmd.Limit < 0 || cmd.Limit > machinediagnostics.MaxReadEvents { + return fmt.Errorf("diagnostics limit must be between 0 and %d", machinediagnostics.MaxReadEvents) + } + if err := machinediagnostics.ValidateCursor(cmd.After); err != nil { + return fmt.Errorf("invalid diagnostics cursor: %w", err) + } + cfg, err := config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + mc, err := workspace.GetMachine(cfg, args) + if err != nil { + return err + } + result, err := fetchDiagnostics(ctx, mc, cmd.After, cmd.Limit, !cmd.NoEvents) + if err != nil { + return err + } + return renderDiagnostics(result, cmd.ResultFormat) +} +func fetchDiagnostics(ctx context.Context, mc client.MachineClient, after string, limit int, includeEvents bool) (MachineDiagnostics, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + result := MachineDiagnostics{SchemaVersion: machinediagnostics.SchemaVersion, Cursor: machinediagnostics.CursorInfo{State: machinediagnostics.CursorNone}} + result.Machine.ID = mc.Machine() + result.Machine.Context = mc.Context() + result.Machine.Provider = mc.Provider() + status, err := mc.Status(ctx, client.StatusOptions{}) + if err != nil { + if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded { + return result, ctx.Err() + } + result.Machine.State = "unknown" + result.Source = CollectionSource{Availability: "unavailable", Freshness: "unknown", ErrorCode: "machine_status_unavailable", Message: "Devsy could not read the provider's machine state. Try again when the provider connection is available."} + return result, nil + } + result.Machine.State = string(status) + if !strings.EqualFold(string(status), string(client.StatusRunning)) { + if status == client.StatusStopped { + result.Source = CollectionSource{Availability: "machine_stopped", Freshness: "unknown"} + } else { + result.Source = CollectionSource{Availability: "unavailable", Freshness: "unknown", ErrorCode: "machine_not_running", Message: "The machine is not running."} + } + return result, nil + } + command := shellescape.Quote(mc.AgentPath()) + " internal agent daemon-diagnostics --limit " + strconv.Itoa(limit) + if after != "" { + command += " --after " + shellescape.Quote(after) + } + stdout := boundedDiagnosticsBuffer{max: maxRemoteDiagnosticsResponse} + stderr := boundedDiagnosticsBuffer{max: maxRemoteDiagnosticsResponse} + err = mc.Command(ctx, client.CommandOptions{Command: command, Stdout: &stdout, Stderr: &stderr}) + if err != nil { + result.Source = CollectionSource{Availability: "unavailable", Freshness: "unknown", ErrorCode: "remote_diagnostics_command_failed", Message: "Devsy could not read remote diagnostics."} + return result, nil + } + var remote machinediagnostics.ReadResponse + if err := json.Unmarshal(stdout.Bytes(), &remote); err != nil { + result.Source = CollectionSource{Availability: "unavailable", Freshness: "unknown", ErrorCode: "remote_diagnostics_command_failed", Message: "Remote diagnostics returned an invalid response."} + return result, nil + } + if remote.SchemaVersion != machinediagnostics.SchemaVersion || remote.Availability == "" || remote.Freshness == "" { + result.Source = CollectionSource{Availability: "unavailable", Freshness: "unknown", ErrorCode: "invalid_diagnostics_response", Message: "Remote diagnostics returned an unsupported or incomplete response."} + return result, nil + } + result.Source = CollectionSource{Availability: string(remote.Availability), Freshness: string(remote.Freshness)} + if remote.Error != nil { + result.Source.ErrorCode = remote.Error.Code + result.Source.Message = remote.Error.Message + } + result.Daemon = remote.Status + if includeEvents { + result.Events = remote.Events + } + result.Cursor = remote.Cursor + return result, nil +} +func renderDiagnostics(result MachineDiagnostics, format string) error { + mode, err := output.ResolveMode(format) + if err != nil { + return err + } + if mode == output.ModeJSON { + return json.NewEncoder(os.Stdout).Encode(result) + } + return renderDiagnosticsText(os.Stdout, result) +} + +func renderDiagnosticsText(w io.Writer, result MachineDiagnostics) error { + _, _ = fmt.Fprintf(w, "Machine %s\nProvider state %s\nDiagnostics %s\n", result.Machine.ID, result.Machine.State, result.Source.Availability) + if result.Source.Message != "" { + _, _ = fmt.Fprintf(w, "Diagnostic detail %s\n", result.Source.Message) + } + if result.Daemon != nil { + lastPatrol := "Not yet observed" + if result.Daemon.LastPatrolAt != nil { + lastPatrol = result.Daemon.LastPatrolAt.Local().Format("2006-01-02 15:04:05") + } + _, _ = fmt.Fprintf(w, "Devsy daemon %s\nSnapshot freshness %s\nLast patrol %s\nWorkspaces %d\n", result.Daemon.Health, result.Source.Freshness, lastPatrol, result.Daemon.WorkspaceCount) + if result.Daemon.LastError != nil { + _, _ = fmt.Fprintf(w, "Last daemon error %s (%s, %s)\n", result.Daemon.LastError.Message, result.Daemon.LastError.Code, result.Daemon.LastError.Timestamp.Format(time.RFC3339)) + } + if result.Daemon.ShutdownCandidate != nil { + eligible := "now" + if result.Daemon.ShutdownCandidate.EligibleAt != nil { + eligible = result.Daemon.ShutdownCandidate.EligibleAt.Local().Format("2006-01-02 15:04:05") + } + _, _ = fmt.Fprintf(w, "Shutdown candidate %s (eligible %s)\n", result.Daemon.ShutdownCandidate.WorkspaceID, eligible) + } + renderWorkspaceDiagnostics(w, result.Daemon.Workspaces) + } + for _, event := range result.Events { + _, _ = fmt.Fprintf(w, "%s %-5s %s %s\n", event.Timestamp.Format(time.RFC3339), event.Level, event.Type, event.Message) + } + return nil +} + +func renderWorkspaceDiagnostics(w io.Writer, workspaces []machinediagnostics.WorkspaceStatus) { + if len(workspaces) == 0 { + return + } + _, _ = fmt.Fprintln(w, "\nWorkspace inactivity:") + tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(tw, "WORKSPACE\tSTATE\tLAST ACTIVITY\tMACHINE SHUTDOWN STATUS") + for _, workspace := range workspaces { + lastActivity := "-" + if workspace.LastActivityAt != nil { + lastActivity = workspace.LastActivityAt.Local().Format("2006-01-02 15:04:05") + } + _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", workspace.ID, workspace.State, lastActivity, workspaceAutoStopDetail(workspace)) + } + _ = tw.Flush() +} + +func workspaceAutoStopDetail(workspace machinediagnostics.WorkspaceStatus) string { + if workspace.BlocksMachineShutdown && workspace.BlockerReason != "" { + if workspace.State == machinediagnostics.WorkspaceActive && workspace.IdleDeadlineAt != nil { + return workspace.BlockerReason + " (" + workspace.IdleDeadlineAt.Local().Format("2006-01-02 15:04:05") + ")" + } + return workspace.BlockerReason + } + if workspace.IdleDeadlineAt != nil { + if workspace.State == machinediagnostics.WorkspaceIdleDue { + return "Eligible now (" + workspace.IdleDeadlineAt.Local().Format("2006-01-02 15:04:05") + ")" + } + return workspace.IdleDeadlineAt.Local().Format("2006-01-02 15:04:05") + } + switch workspace.State { + case machinediagnostics.WorkspaceBusy: + return "Delayed while workspace is busy" + case machinediagnostics.WorkspaceNotConfigured: + return "Auto-stop is not configured" + case machinediagnostics.WorkspaceInvalidConfig: + return "Workspace configuration is invalid" + case machinediagnostics.WorkspaceNotRunning: + return "Workspace state is unavailable" + default: + return "-" + } +} diff --git a/cmd/machine/diagnostics_test.go b/cmd/machine/diagnostics_test.go new file mode 100644 index 000000000..9ea111497 --- /dev/null +++ b/cmd/machine/diagnostics_test.go @@ -0,0 +1,133 @@ +package machine + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/client" + "github.com/devsy-org/devsy/pkg/machinediagnostics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeMachineClient struct { + client.MachineClient + status client.Status + statusErr error + commandErr error + command string + commandCalls int + response string +} + +func (f *fakeMachineClient) Machine() string { return "machine-1" } +func (f *fakeMachineClient) Context() string { return "default" } +func (f *fakeMachineClient) Provider() string { return "fake" } +func (f *fakeMachineClient) AgentPath() string { return "/usr/local/bin/devsy" } +func (f *fakeMachineClient) Status(context.Context, client.StatusOptions) (client.Status, error) { + return f.status, f.statusErr +} +func (f *fakeMachineClient) Command(_ context.Context, options client.CommandOptions) error { + f.commandCalls++ + f.command = options.Command + if f.commandErr != nil { + return f.commandErr + } + _, _ = options.Stdout.Write([]byte(f.response)) + return nil +} + +func TestBoundedDiagnosticsBuffer(t *testing.T) { + buffer := boundedDiagnosticsBuffer{max: 4} + _, err := buffer.Write([]byte("test")) + require.NoError(t, err) + _, err = buffer.Write([]byte("x")) + require.ErrorContains(t, err, "exceeds") + assert.Equal(t, "test", buffer.String()) +} + +func TestFetchDiagnosticsSkipsRemoteCommandForStoppedMachine(t *testing.T) { + machine := &fakeMachineClient{status: client.StatusStopped} + result, err := fetchDiagnostics(context.Background(), machine, "", 20, true) + require.NoError(t, err) + assert.Equal(t, "machine_stopped", result.Source.Availability) + assert.Zero(t, machine.commandCalls) +} + +func TestFetchDiagnosticsReadsRemoteResponse(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + response, err := json.Marshal(machinediagnostics.ReadResponse{ + SchemaVersion: machinediagnostics.SchemaVersion, + Availability: machinediagnostics.AvailabilityAvailable, + Freshness: machinediagnostics.FreshnessFresh, + Status: &machinediagnostics.Status{Health: machinediagnostics.DaemonHealthy}, + Events: []machinediagnostics.Event{{SessionID: "session", Sequence: 1, Timestamp: now, Type: machinediagnostics.EventDaemonReady, Message: "ready"}}, + Cursor: machinediagnostics.CursorInfo{State: machinediagnostics.CursorOK, Next: machinediagnostics.EncodeCursor("session", 1)}, + }) + require.NoError(t, err) + machine := &fakeMachineClient{status: client.StatusRunning, response: string(response)} + result, err := fetchDiagnostics(context.Background(), machine, "", 20, true) + require.NoError(t, err) + assert.Equal(t, "available", result.Source.Availability) + assert.Equal(t, machinediagnostics.DaemonHealthy, result.Daemon.Health) + assert.Len(t, result.Events, 1) + assert.Contains(t, machine.command, "daemon-diagnostics --limit 20") +} + +func TestFetchDiagnosticsRetainsAvailabilityWhenRemoteCommandFails(t *testing.T) { + machine := &fakeMachineClient{status: client.StatusRunning, commandErr: errors.New("ssh lost")} + result, err := fetchDiagnostics(context.Background(), machine, "", 20, true) + require.NoError(t, err) + assert.Equal(t, "unavailable", result.Source.Availability) + assert.Equal(t, "remote_diagnostics_command_failed", result.Source.ErrorCode) +} + +func TestFetchDiagnosticsRejectsIncompleteResponse(t *testing.T) { + machine := &fakeMachineClient{status: client.StatusRunning, response: `{}`} + result, err := fetchDiagnostics(context.Background(), machine, "", 20, true) + require.NoError(t, err) + assert.Equal(t, "unavailable", result.Source.Availability) + assert.Equal(t, "invalid_diagnostics_response", result.Source.ErrorCode) +} + +func TestFetchDiagnosticsReportsProviderFailureWithoutEndingCollection(t *testing.T) { + machine := &fakeMachineClient{statusErr: errors.New("provider connection lost")} + result, err := fetchDiagnostics(context.Background(), machine, "", 20, true) + require.NoError(t, err) + assert.Equal(t, "unknown", result.Machine.State) + assert.Equal(t, "machine_status_unavailable", result.Source.ErrorCode) + assert.Zero(t, machine.commandCalls) +} + +func TestRenderDiagnosticsTextIncludesWorkspaceInactivityDetails(t *testing.T) { + now := time.Date(2026, 9, 19, 1, 30, 0, 0, time.Local) + deadline := now.Add(30 * time.Minute) + var output bytes.Buffer + renderWorkspaceDiagnostics(&output, []machinediagnostics.WorkspaceStatus{ + {ID: "api", State: machinediagnostics.WorkspaceActive, LastActivityAt: &now, IdleDeadlineAt: &deadline}, + {ID: "docs", State: machinediagnostics.WorkspaceBusy}, + {ID: "preview", State: machinediagnostics.WorkspaceNotConfigured}, + {ID: "broken", State: machinediagnostics.WorkspaceInvalidConfig}, + }) + text := output.String() + assert.Contains(t, text, "Workspace inactivity:") + assert.Contains(t, text, "MACHINE SHUTDOWN STATUS") + assert.Contains(t, text, "api") + assert.Contains(t, text, deadline.Format("2006-01-02 15:04:05")) + assert.Contains(t, text, "Delayed while workspace is busy") + assert.Contains(t, text, "Auto-stop is not configured") + assert.Contains(t, text, "Workspace configuration is invalid") +} + +func TestWorkspaceAutoStopDetailMarksPastDeadlineEligibleNow(t *testing.T) { + deadline := time.Now().Add(-time.Minute) + detail := workspaceAutoStopDetail(machinediagnostics.WorkspaceStatus{ + State: machinediagnostics.WorkspaceIdleDue, + IdleDeadlineAt: &deadline, + }) + assert.Contains(t, detail, "Eligible now") +} diff --git a/cmd/machine/logs.go b/cmd/machine/logs.go new file mode 100644 index 000000000..3910ba793 --- /dev/null +++ b/cmd/machine/logs.go @@ -0,0 +1,92 @@ +package machine + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/devsy-org/devsy/cmd/flags" + "github.com/devsy-org/devsy/pkg/config" + cliflags "github.com/devsy-org/devsy/pkg/flags" + "github.com/devsy-org/devsy/pkg/machinediagnostics" + "github.com/devsy-org/devsy/pkg/output" + "github.com/devsy-org/devsy/pkg/workspace" + "github.com/spf13/cobra" +) + +type LogsCmd struct { + *flags.GlobalFlags + After string + Limit int + Follow bool +} + +func NewLogsCmd(globalFlags *flags.GlobalFlags) *cobra.Command { + cmd := &LogsCmd{GlobalFlags: globalFlags} + c := &cobra.Command{Use: "logs [name]", Short: "Show Devsy machine diagnostic events", RunE: func(c *cobra.Command, args []string) error { return cmd.Run(c.Context(), args) }} + cliflags.Add(c, cliflags.String(&cmd.After, "after", "", "An opaque diagnostics cursor"), cliflags.Int(&cmd.Limit, "limit", machinediagnostics.DefaultReadEvents, "Maximum diagnostic events")) + c.Flags().BoolVarP(&cmd.Follow, "follow", "f", false, "Poll for new diagnostic events until interrupted") + return c +} +func (cmd *LogsCmd) Run(ctx context.Context, args []string) error { + if cmd.Limit < 0 || cmd.Limit > machinediagnostics.MaxReadEvents { + return fmt.Errorf("diagnostics limit must be between 0 and %d", machinediagnostics.MaxReadEvents) + } + if err := machinediagnostics.ValidateCursor(cmd.After); err != nil { + return fmt.Errorf("invalid diagnostics cursor: %w", err) + } + cfg, err := config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + mc, err := workspace.GetMachine(cfg, args) + if err != nil { + return err + } + mode, err := output.ResolveMode(cmd.ResultFormat) + if err != nil { + return err + } + for { + result, err := fetchDiagnostics(ctx, mc, cmd.After, cmd.Limit, true) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + if mode == output.ModeJSON { + if err := json.NewEncoder(os.Stdout).Encode(result); err != nil { + return err + } + } else { + if result.Source.Availability != string(machinediagnostics.AvailabilityAvailable) { + _, _ = fmt.Fprintf(os.Stderr, "Diagnostics: %s %s\n", result.Source.Availability, result.Source.Message) + } + if result.Cursor.State == machinediagnostics.CursorGap || result.Cursor.State == machinediagnostics.CursorReset { + _, _ = fmt.Fprintf(os.Stderr, "Diagnostic history: %s (%s)\n", result.Cursor.State, result.Cursor.Reason) + } + for _, event := range result.Events { + _, _ = fmt.Fprintf(os.Stdout, "%s %-5s %-35s %s\n", event.Timestamp.Format(time.RFC3339), event.Level, event.Type, event.Message) + } + } + if !cmd.Follow { + return nil + } + if result.Cursor.State == machinediagnostics.CursorReset { + cmd.After = "" + } + if result.Cursor.Next != "" { + cmd.After = result.Cursor.Next + } + timer := time.NewTimer(5 * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil + case <-timer.C: + } + } +} diff --git a/cmd/machine/machine.go b/cmd/machine/machine.go index 3bebdab71..ecf1ba796 100644 --- a/cmd/machine/machine.go +++ b/cmd/machine/machine.go @@ -21,5 +21,7 @@ func NewMachineCmd(flags *flags.GlobalFlags) *cobra.Command { machineCmd.AddCommand(NewCreateCmd(flags)) machineCmd.AddCommand(NewInspectCmd(flags)) machineCmd.AddCommand(NewDescribeCmd(flags)) + machineCmd.AddCommand(NewDiagnosticsCmd(flags)) + machineCmd.AddCommand(NewLogsCmd(flags)) return machineCmd } diff --git a/cmd/workspace/delete.go b/cmd/workspace/delete.go index a72d9c3e6..a3f6aec56 100644 --- a/cmd/workspace/delete.go +++ b/cmd/workspace/delete.go @@ -93,16 +93,16 @@ func (cmd *DeleteCmd) Run(cobraCmd *cobra.Command, args []string) error { ctx, reporter, status.Operation{Phase: status.PhaseDeletingWorkspace}, - func(context.Context) error { + func(ctx context.Context) error { var err error devsyConfig, err = cmd.loadConfig() if err != nil { return err } if len(args) <= 1 { - return cmd.deleteSingle(ctx, devsyConfig, args) + return cmd.deleteSingle(status.WithReporter(ctx, reporter), devsyConfig, args) } - return cmd.deleteMultiple(ctx, devsyConfig, args) + return cmd.deleteMultiple(status.WithReporter(ctx, reporter), devsyConfig, args) }, ) if devsyConfig == nil { diff --git a/cmd/workspace/stop.go b/cmd/workspace/stop.go index 29f0b9c6d..424615df2 100644 --- a/cmd/workspace/stop.go +++ b/cmd/workspace/stop.go @@ -33,27 +33,7 @@ func NewStopCmd(flags *flags.GlobalFlags) *cobra.Command { Use: "stop [flags] [workspace-path|workspace-name]", Short: "Stop a workspace", RunE: func(cobraCmd *cobra.Command, args []string) error { - ctx := cobraCmd.Context() - devsyConfig, err := config.LoadConfig(cmd.Context, cmd.Provider) - if err != nil { - return err - } - - err = clientimplementation.DecodePlatformOptionsFromEnv(&cmd.Platform) - if err != nil { - return fmt.Errorf("decode platform options: %w", err) - } - - client, err := workspace2.Get(ctx, workspace2.GetOptions{ - DevsyConfig: devsyConfig, - Args: args, - Owner: cmd.Owner, - }) - if err != nil { - return err - } - - return cmd.Run(ctx, devsyConfig, client) + return cmd.runArgs(cobraCmd.Context(), args) }, ValidArgsFunction: func(rootCmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return completion.GetWorkspaceSuggestions( @@ -89,11 +69,48 @@ func (cmd *StopCmd) Run( reporter, status.Operation{Phase: status.PhaseStoppingWorkspace}, func(ctx context.Context) error { - return cmd.run(ctx, devsyConfig, client) + return cmd.run(status.WithReporter(ctx, reporter), devsyConfig, client) }, ) } +func (cmd *StopCmd) runArgs(ctx context.Context, args []string) error { + reporter, err := newWorkspaceStatusReporter( + cmd.ResultFormat, os.Stdout, cmd.Verbosity > 0 || cmd.Debug, + ) + if err != nil { + return err + } + return status.Run(status.WithReporter(ctx, reporter), reporter, + status.Operation{Phase: status.PhaseStoppingWorkspace}, func(ctx context.Context) error { + var devsyConfig *config.Config + var client client2.BaseWorkspaceClient + err := status.RunStep( + ctx, status.PhaseStoppingWorkspace, "Loading workspace", + func(ctx context.Context) error { + var err error + devsyConfig, err = config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + if err := clientimplementation.DecodePlatformOptionsFromEnv( + &cmd.Platform, + ); err != nil { + return fmt.Errorf("decode platform options: %w", err) + } + client, err = workspace2.Get(ctx, workspace2.GetOptions{ + DevsyConfig: devsyConfig, Args: args, Owner: cmd.Owner, + }) + return err + }, + ) + if err != nil { + return err + } + return cmd.run(ctx, devsyConfig, client) + }) +} + func (cmd *StopCmd) run( ctx context.Context, devsyConfig *config.Config, @@ -101,7 +118,12 @@ func (cmd *StopCmd) run( ) error { // lock workspace if !cmd.Platform.Enabled { - err := client.Lock(ctx) + err := status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Waiting for workspace lock", + client.Lock, + ) if err != nil { return err } @@ -109,7 +131,17 @@ func (cmd *StopCmd) run( } // get instance status - instanceStatus, err := client.Status(ctx, client2.StatusOptions{}) + var instanceStatus client2.Status + err := status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Checking workspace status", + func(ctx context.Context) error { + var err error + instanceStatus, err = client.Status(ctx, client2.StatusOptions{}) + return err + }, + ) if err != nil { return err } else if instanceStatus != client2.StatusRunning { @@ -117,7 +149,17 @@ func (cmd *StopCmd) run( } // stop if single machine provider - wasStopped, err := cmd.stopSingleMachine(ctx, client, devsyConfig) + var wasStopped bool + err = status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Checking shared machine", + func(ctx context.Context) error { + var err error + wasStopped, err = cmd.stopSingleMachine(ctx, client, devsyConfig) + return err + }, + ) if err != nil { return err } else if wasStopped { @@ -126,7 +168,14 @@ func (cmd *StopCmd) run( } // stop environment - err = client.Stop(ctx, client2.StopOptions{}) + err = status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Stopping workspace resources", + func(ctx context.Context) error { + return client.Stop(ctx, client2.StopOptions{}) + }, + ) if err != nil { return err } @@ -171,9 +220,16 @@ func (cmd *StopCmd) stopSingleMachine( } // stop the machine - err = machineClient.Stop(ctx, client2.StopOptions{}) + err = status.RunStep( + ctx, + status.PhaseStoppingWorkspace, + "Stopping machine", + func(ctx context.Context) error { + return machineClient.Stop(ctx, client2.StopOptions{}) + }, + ) if err != nil { - return false, fmt.Errorf("delete machine: %w", err) + return false, fmt.Errorf("stop machine: %w", err) } log.Debugf("stopped workspace: workspace=%s", client.Workspace()) diff --git a/desktop/e2e/fixtures/mock-devsy.cjs b/desktop/e2e/fixtures/mock-devsy.cjs index 1b8dfea54..da0084f67 100755 --- a/desktop/e2e/fixtures/mock-devsy.cjs +++ b/desktop/e2e/fixtures/mock-devsy.cjs @@ -283,6 +283,8 @@ function handleUp(args) { return } + workspaceStatus("building_image", "started", "Building workspace") + const complete = () => { workspaceStatus("resolving_config", "started") workspaceStatus("resolving_config", "succeeded") workspaceStatus("building_image", "started") @@ -292,6 +294,9 @@ function handleUp(args) { workspaceStatus("ready", "succeeded") materializeWorkspace(wsId, source, providerFlag, ideFlag, "Running") process.exit(0) + } + if (wsId === "lifecycleprobe") setTimeout(complete, 2000) + else complete() } // Handlers for `workspace task `, backing the up --detach flow above. @@ -455,14 +460,16 @@ function handleTask(args) { function handleStop(args) { const { positional } = parseArgs(args) const wsId = positional[0] - out("Stopping workspace...") - out("Workspace stopped.") - const ws = state.workspaces.find((w) => w.id === wsId) - if (ws) { - ws.status = "Stopped" - saveState(state) + workspaceStatus("stopping_workspace", "started", "Stopping workspace resources") + const complete = () => { + const latest = loadState() + const ws = latest.workspaces.find((w) => w.id === wsId) + if (ws) { ws.status = "Stopped"; saveState(latest) } + workspaceStatus("stopping_workspace", "succeeded") + process.exit(0) } - process.exit(0) + if (wsId === "lifecycleprobe") setTimeout(complete, 2000) + else complete() } function handleStart(args) { @@ -484,7 +491,8 @@ function handleDelete(args) { const { positional } = parseArgs(args) const wsId = positional[0] // *probe suffix delays deletion so lifecycle tests can observe in-flight state. - const deleteMs = /probe$/.test(wsId) ? 1500 : 0 + const deleteMs = wsId === "lifecycleprobe" ? 8000 : /probe$/.test(wsId) ? 1500 : 0 + workspaceStatus("deleting_workspace", "started", "Removing workspace resources") setTimeout(() => { out("Deleting workspace...") out("Workspace deleted.") diff --git a/desktop/e2e/workspace-progress.e2e.ts b/desktop/e2e/workspace-progress.e2e.ts new file mode 100644 index 000000000..331680077 --- /dev/null +++ b/desktop/e2e/workspace-progress.e2e.ts @@ -0,0 +1,64 @@ +import type { ElectronApplication, Page } from "@playwright/test" +import { expect, test } from "@playwright/test" +import { launchApp, resetMockState } from "./electron-app.js" +let app: ElectronApplication +let page: Page +const id = "lifecycleprobe" +async function invoke(channel: string, args?: Record) { + return page.evaluate( + ({ channel, args }) => window.electronAPI.invoke(channel, args), + { channel, args }, + ) +} +async function finished() { + await expect + .poll( + async () => { + const snapshot = (await invoke("workspace_snapshot")) as { + jobs: Record + } + return snapshot.jobs[id]?.state + }, + { timeout: 30000 }, + ) + .toBe("succeeded") +} +test.beforeAll(async () => { + resetMockState() + ;({ app, page } = await launchApp()) + await page.click('[data-sidebar="sidebar"] a[href="#/workspaces"]') +}) +test.afterAll(async () => { + await app?.close() +}) +test("shares lifecycle progress across actions, navigation and window reload", async () => { + test.setTimeout(120000) + await invoke("workspace_up", { + source: "https://example.com/lifecycleprobe.git", + workspaceId: id, + }) + await finished() + const row = () => page.locator("tr").filter({ hasText: id }) + await expect(row()).toContainText("Running") + for (const [action, label] of [ + ["stop", "Stopping"], + ["rebuild", "Rebuilding"], + ["reset", "Resetting"], + ]) { + await invoke(`workspace_${action}`, { workspaceId: id }) + await expect(row()).toContainText(label) + await finished() + } + await invoke("workspace_up", { source: id }) + await finished() + await invoke("workspace_delete", { workspaceId: id }) + await expect(row()).toContainText("Deleting") + await page.click('[data-sidebar="sidebar"] a[href="#/machines"]') + await page.click('[data-sidebar="sidebar"] a[href="#/workspaces"]') + await expect(row()).toContainText("Deleting") + await page.reload() + await page.locator('[data-sidebar="sidebar"]').waitFor() + await expect(row()).toContainText("Deleting") + await finished() + await expect(row()).toHaveCount(0) +}) diff --git a/desktop/src/main/__tests__/ipc-up-tasks.test.ts b/desktop/src/main/__tests__/ipc-up-tasks.test.ts index 37eed1394..3900e494a 100644 --- a/desktop/src/main/__tests__/ipc-up-tasks.test.ts +++ b/desktop/src/main/__tests__/ipc-up-tasks.test.ts @@ -1,4 +1,5 @@ // @vitest-environment node +import { WorkspaceJobs } from "../workspace-jobs.js" import { EventEmitter } from "node:events" import { beforeEach, describe, expect, it, vi } from "vitest" @@ -22,16 +23,20 @@ vi.mock("../analytics.js", () => ({ const { registerIpcHandlers } = await import("../ipc.js") -function invokeUp(workspaceId: string): Promise { +async function invokeUp(workspaceId: string): Promise { const handler = handlers.get("workspace_up") if (!handler) throw new Error("workspace_up not registered") - return handler({}, { source: workspaceId, workspaceId }) as Promise + const id = (await handler({}, { source: workspaceId, workspaceId })) as string + await new Promise((resolve) => setTimeout(resolve, 10)) + return id } -function invokeStop(workspaceId: string): Promise { +async function invokeStop(workspaceId: string): Promise { const handler = handlers.get("workspace_stop") if (!handler) throw new Error("workspace_stop not registered") - return handler({}, { workspaceId }) as Promise + const id = (await handler({}, { workspaceId })) as string + await new Promise((resolve) => setTimeout(resolve, 10)) + return id } /** A child that reports itself still alive, so cancel must await its exit. */ @@ -78,6 +83,7 @@ function setup( }, ), cancelFor: vi.fn(async () => undefined), + runRaw: vi.fn(async () => statusEnvelope("building_image")), } let stream: { onLine: (line: string, s: "stdout" | "stderr") => void @@ -93,7 +99,10 @@ function setup( }, isDestroyed: () => false, } + const jobs = new WorkspaceJobs() + jobs.setRefresh(onWorkspaceStopComplete) const deps = { + workspaceJobs: jobs, cli, state: { workspaceContext: () => "ctx", @@ -114,6 +123,7 @@ function setup( return { cli, calls, + jobs, api, sent, stream: () => stream, @@ -138,11 +148,11 @@ describe("workspace_up detached task tracking", () => { vi.clearAllMocks() }) - it("cancels the prior task before submitting a replacement", async () => { + it("stop cancels an active start before launching", async () => { const { calls } = setup() await invokeUp("ws-1") - await invokeUp("ws-1") + await invokeStop("ws-1") const cancels = calls.filter((a) => a.includes("cancel")) expect(cancels).toEqual([["workspace", "task", "cancel", "task-1"]]) @@ -158,7 +168,7 @@ describe("workspace_up detached task tracking", () => { if (!active) throw new Error("stream callback was not registered") active.onLine(`token=${secret}`, "stderr") active.onExit(0) - await Promise.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) const payload = JSON.stringify(sent) expect(payload).not.toContain(secret) @@ -178,7 +188,7 @@ describe("workspace_up detached task tracking", () => { "stderr", ) active.onExit(0) - await Promise.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) const payload = JSON.stringify(sent) expect(payload).not.toContain("git-token") @@ -216,7 +226,7 @@ describe("workspace_up detached task tracking", () => { } }) - it("serializes concurrent submissions so neither task is left orphaned", async () => { + it("rejects concurrent submissions without orphaning a task", async () => { let seq = 0 const { calls } = setup({ run: async (args) => { @@ -230,10 +240,17 @@ describe("workspace_up detached task tracking", () => { }, }) - await Promise.all([invokeUp("ws-1"), invokeUp("ws-1")]) + const submissions = await Promise.allSettled([ + invokeUp("ws-1"), + invokeUp("ws-1"), + ]) + expect( + submissions.filter((result) => result.status === "rejected"), + ).toHaveLength(1) const cancels = calls.filter((a) => a.includes("cancel")) - expect(cancels).toEqual([["workspace", "task", "cancel", "task-1"]]) + expect(cancels).toEqual([]) + expect(calls.filter((args) => args.includes("--detach"))).toHaveLength(1) }) it("keeps the task cancellable when cancellation fails", async () => { @@ -251,7 +268,7 @@ describe("workspace_up detached task tracking", () => { await invokeUp("ws-1") - await invokeUp("ws-1") + await invokeStop("ws-1") expect(calls.filter((a) => a.includes("--detach"))).toHaveLength(1) failCancel = false @@ -296,7 +313,9 @@ describe("workspace_up detached task tracking", () => { "stdout", ) - expect(sent.filter((s) => s.channel === "workspace-status")[0].payload).toMatchObject({ + expect( + sent.filter((s) => s.channel === "workspace-status")[0].payload, + ).toMatchObject({ workspaceId: "ws-1", phase: "building_image", state: "succeeded", @@ -340,6 +359,7 @@ describe("workspace_up detached task tracking", () => { "stdout", ) stream()?.onExit(0) + await new Promise((resolve) => setTimeout(resolve, 10)) await invokeUp("ws-1") expect(calls.filter((a) => a.includes("cancel"))).toEqual([]) @@ -354,6 +374,7 @@ describe("workspace_up detached task tracking", () => { "stdout", ) stream()?.onExit(1) + await new Promise((resolve) => setTimeout(resolve, 10)) await invokeUp("ws-1") expect(calls.filter((a) => a.includes("cancel"))).toEqual([]) @@ -377,13 +398,12 @@ describe("workspace_up detached task tracking", () => { await invokeUp("ws-1") stream()?.onExit(1, { code: "boom", message: "follower died" }) - expect(sent.filter((s) => s.channel === "workspace-status")[0].payload).toMatchObject({ - workspaceId: "ws-1", - phase: "failed", - state: "failed", - error: { code: "boom", message: "follower died" }, - }) - await invokeUp("ws-1") + expect( + sent + .filter((s) => s.channel === "workspace-status") + .some((s) => s.payload.state === "failed"), + ).toBe(false) + await invokeStop("ws-1") expect(calls.filter((a) => a.includes("cancel"))).toEqual([ ["workspace", "task", "cancel", "task-1"], @@ -396,7 +416,10 @@ describe("workspace_up detached task tracking", () => { await invokeStop("ws-1") stream()?.onExit(1) await vi.waitFor(() => - expect(onWorkspaceStopComplete).toHaveBeenCalledWith("ws-1"), + expect(onWorkspaceStopComplete).toHaveBeenCalledWith( + "ws-1", + expect.objectContaining({ error: expect.any(String) }), + ), ) }) }) diff --git a/desktop/src/main/__tests__/ipc-workspace-jobs.test.ts b/desktop/src/main/__tests__/ipc-workspace-jobs.test.ts index 130dd59b5..dea3e7f3b 100644 --- a/desktop/src/main/__tests__/ipc-workspace-jobs.test.ts +++ b/desktop/src/main/__tests__/ipc-workspace-jobs.test.ts @@ -3,125 +3,171 @@ import { EventEmitter } from "node:events" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { ProviderJobs } from "../provider-jobs.js" import { WorkspaceJobs } from "../workspace-jobs.js" - -const handlers = new Map unknown>() - +const handlers = new Map any>() vi.mock("electron", () => ({ app: { getPath: () => "/tmp", getVersion: () => "0.0.0" }, dialog: {}, ipcMain: { - handle: (channel: string, fn: (...args: unknown[]) => unknown) => { - handlers.set(channel, fn) - }, + handle: (channel: string, fn: (...args: any[]) => any) => + handlers.set(channel, fn), on: () => undefined, }, })) - vi.mock("../analytics.js", () => ({ - hashWorkspaceRef: (v: string) => v, + hashWorkspaceRef: (value: string) => value, trackEvent: () => undefined, })) - const { registerIpcHandlers } = await import("../ipc.js") - -function fakeChild() { - const child = new EventEmitter() as EventEmitter & { - exitCode: number | null - signalCode: string | null - kill: () => void - } - child.exitCode = null - child.signalCode = null - child.kill = () => undefined - return child -} - -function setup(exitCode = 0) { - const workspaceJobs = new WorkspaceJobs() - const sent: Array<{ channel: string; payload: unknown }> = [] +function setup() { + const jobs = new WorkspaceJobs() + let exit!: (code: number) => void + let line!: (line: string, stream: "stdout") => void const cli = { - run: vi.fn(async () => []), - runRaw: vi.fn(async () => ""), - runStreaming: vi.fn( - async ( - _cliArgs: string[], - _onLine: (line: string, stream: "stdout" | "stderr") => void, - onExit: (code: number, cliError?: unknown) => void, - ) => { - const child = fakeChild() - setTimeout(() => { - onExit(exitCode, exitCode === 0 ? undefined : { message: "boom" }) - }, 0) - return child - }, + run: vi.fn(async () => ({ id: "task-1" })), + runRaw: vi.fn( + async () => + '{"kind":"status","schemaVersion":1,"phase":"building_image","state":"started"}', ), - cancelFor: vi.fn(async () => undefined), + runStreaming: vi.fn(async (_args, onLine, onExit) => { + exit = onExit + line = onLine + return Object.assign(new EventEmitter(), { + exitCode: null, + signalCode: null, + kill: vi.fn(), + }) + }), + cancelFor: vi.fn(async () => {}), } - const deps = { + const send = vi.fn() + const actions = registerIpcHandlers({ cli, - state: { workspaceContext: () => "ctx", providerList: () => [] }, + state: { + workspaceContext: () => "ctx", + providerList: () => [], + workspaceList: () => [], + }, logStore: { - createLogFile: () => "/tmp/log.txt", + createLogFile: () => "/tmp/log", appendLog: () => true, - closeLog: async () => undefined, - onDrain: async () => undefined, + closeLog: async () => {}, + onDrain: async () => {}, }, - pty: { cancelFor: vi.fn(async () => undefined) }, - getMainWindow: () => ({ - webContents: { - send: (channel: string, payload: unknown) => sent.push({ channel, payload }), - }, - }), + pty: { cancelFor: vi.fn(async () => {}) }, + getMainWindow: () => ({ webContents: { send } }), providerJobs: new ProviderJobs(), - workspaceJobs, + workspaceJobs: jobs, + } as any) + return { + jobs, + cli, + send, + actions, + exit: (code: number) => exit(code), + line: (text: string) => line(text, "stdout"), } - // biome-ignore lint/suspicious/noExplicitAny: partial test doubles - registerIpcHandlers(deps as any) - return { workspaceJobs, sent } } - -function invoke(channel: string, args: Record) { - const handler = handlers.get(channel) - if (!handler) throw new Error(`${channel} not registered`) - return handler({}, args) +function invoke(channel: string, args = { workspaceId: "ws" }) { + return handlers.get(channel)!({}, args) +} +async function flush() { + for (let i = 0; i < 20; i++) await Promise.resolve() } - -describe("workspace delete job lifecycle over IPC", () => { - beforeEach(() => { - handlers.clear() - vi.clearAllMocks() - vi.useFakeTimers() +beforeEach(() => { + handlers.clear() + vi.clearAllMocks() + vi.useFakeTimers() +}) +afterEach(() => vi.useRealTimers()) +describe("workspace lifecycle IPC", () => { + it.each(["delete", "stop", "rebuild", "reset"])( + "acknowledges %s while shutdown is blocked", + async (action) => { + const ctx = setup() + let release!: () => void + ctx.cli.cancelFor.mockImplementation( + () => + new Promise((resolve) => { + release = resolve + }), + ) + const commandId = invoke(`workspace_${action}`) + expect(typeof commandId).toBe("string") + expect(ctx.jobs.get("ws")).toMatchObject({ + commandId, + state: "running", + phase: "Closing connections", + }) + await flush() + expect(ctx.cli.runStreaming).not.toHaveBeenCalled() + release() + await flush() + ctx.exit(0) + await flush() + expect(ctx.jobs.get("ws")?.state).toBe("succeeded") + }, + ) + it("records preparation failure without launching the CLI", async () => { + const ctx = setup() + ctx.cli.cancelFor.mockRejectedValue(new Error("shutdown failed")) + invoke("workspace_delete") + await flush() + expect(ctx.jobs.get("ws")).toMatchObject({ + state: "failed", + error: "shutdown failed", + }) + expect(ctx.cli.runStreaming).not.toHaveBeenCalled() + }) + it("rejects duplicate deletes", () => { + setup() + invoke("workspace_delete") + expect(() => invoke("workspace_delete")).toThrow("already in progress") + }) + it("reconciles CLI failures and keeps their error", async () => { + const ctx = setup() + const refresh = vi.fn(async () => {}) + ctx.jobs.setRefresh(refresh) + invoke("workspace_delete") + await flush() + ctx.exit(1) + await flush() + expect(ctx.jobs.get("ws")).toMatchObject({ + state: "failed", + error: "Command exited with code 1", + }) + expect(refresh).toHaveBeenCalledOnce() }) - - afterEach(() => { - vi.useRealTimers() + it("does not interpret a child phase success as completion", async () => { + const ctx = setup() + invoke("workspace_rebuild") + await flush() + ctx.line( + '{"kind":"status","schemaVersion":1,"phase":"building_image","state":"succeeded"}', + ) + expect(ctx.jobs.get("ws")?.state).toBe("running") }) - - it("shows a deleting job while the command runs, then clears it", async () => { - const { workspaceJobs, sent } = setup(0) - - // workspace_delete resolves as soon as the CLI command is launched, well - // before it exits — the job must already be visible at that point. - await invoke("workspace_delete", { workspaceId: "ws1" }) - expect(workspaceJobs.get("ws1")).toEqual({ activity: "deleting" }) - - // Deterministically flush the mock's exit callback instead of racing it - // against a fixed wall-clock wait. - await vi.runAllTimersAsync() - - expect(workspaceJobs.get("ws1")).toBeUndefined() - expect(sent.filter((entry) => entry.channel === "workspace-status").map((entry) => entry.payload)).toEqual([ - { commandId: expect.any(String), workspaceId: "ws1", phase: "deleting_workspace", state: "started" }, - { commandId: expect.any(String), workspaceId: "ws1", phase: "deleting_workspace", state: "succeeded" }, - ]) + it("checks the detached task when its follower exits cleanly", async () => { + const ctx = setup() + await handlers.get("workspace_up")!({}, { source: "ws", commandId: "up" }) + await flush() + ctx.exit(0) + await flush() + expect(ctx.cli.runRaw).toHaveBeenCalled() + expect(ctx.jobs.get("ws")?.state).toBe("running") + ctx.cli.runRaw.mockResolvedValue( + '{"kind":"result","outcome":"success","containerId":"c","remoteUser":"u","remoteWorkspaceFolder":"/w"}', + ) + await vi.advanceTimersByTimeAsync(3000) + await flush() + expect(ctx.jobs.get("ws")?.state).toBe("succeeded") }) - - it("retains the failure when the delete command fails", async () => { - const { workspaceJobs } = setup(1) - - await invoke("workspace_delete", { workspaceId: "ws1" }) - await vi.runAllTimersAsync() - - expect(workspaceJobs.get("ws1")?.error).toBe("boom") + it("uses the same registry for tray stop", async () => { + const ctx = setup() + const completion = ctx.actions.workspaceActions.stop("ws") + expect(ctx.jobs.get("ws")?.activity).toBe("stopping") + await flush() + ctx.exit(0) + await completion + expect(ctx.jobs.get("ws")?.state).toBe("succeeded") }) }) diff --git a/desktop/src/main/__tests__/machine-diagnostics-manager.test.ts b/desktop/src/main/__tests__/machine-diagnostics-manager.test.ts new file mode 100644 index 000000000..8a3f7a0e2 --- /dev/null +++ b/desktop/src/main/__tests__/machine-diagnostics-manager.test.ts @@ -0,0 +1,54 @@ +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it, vi } from "vitest" +import type { CliRunner } from "../cli.js" +import { MachineDiagnosticsManager } from "../machine-diagnostics-manager.js" +import { MachineDiagnosticsStore } from "../machine-diagnostics-store.js" + +describe("MachineDiagnosticsManager", () => { + it("coalesces concurrent refreshes for the same machine", async () => { + const runRaw = vi.fn().mockResolvedValue(JSON.stringify({ + schemaVersion: 1, + machine: { id: "machine", context: "default", provider: "test", state: "Running" }, + source: { availability: "available", freshness: "fresh" }, + events: [], + cursor: { state: "none" }, + })) + const manager = new MachineDiagnosticsManager( + { runRaw } as unknown as CliRunner, + new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))), + ) + const key = { context: "default", machineId: "machine" } + const [first, second] = await Promise.all([manager.refresh(key), manager.refresh(key)]) + expect(runRaw).toHaveBeenCalledTimes(1) + expect(runRaw.mock.calls[0][0]).toEqual(expect.arrayContaining(["--context", "default"])) + expect(first).toEqual(second) + }) + + it("does not recreate a deleted cache when an older collection completes", async () => { + let resolve!: (value: string) => void + const runRaw = vi.fn(() => new Promise((done) => { resolve = done })) + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + const manager = new MachineDiagnosticsManager({ runRaw } as unknown as CliRunner, store) + const key = { context: "default", machineId: "machine" } + const pending = manager.refresh(key) + manager.delete(key) + resolve(JSON.stringify({ schemaVersion: 1, machine: { id: "machine", context: "default" }, source: { availability: "available", freshness: "fresh" }, cursor: { state: "none" } })) + await expect(pending).rejects.toThrow("superseded") + expect(store.get(key)).toBeNull() + }) + + it("retains the last good snapshot after a collection failure", async () => { + const success = JSON.stringify({ schemaVersion: 1, machine: { id: "machine", context: "default", provider: "test", state: "Running" }, source: { availability: "available", freshness: "fresh" }, events: [], cursor: { state: "none" } }) + const runRaw = vi.fn().mockResolvedValueOnce(success).mockRejectedValueOnce(new Error("network unavailable")).mockResolvedValueOnce(success) + const manager = new MachineDiagnosticsManager({ runRaw } as unknown as CliRunner, new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-")))) + const key = { context: "default", machineId: "machine" } + await manager.refresh(key) + const retained = await manager.refresh(key) + expect(retained.lastCollectionError).toBe("network unavailable") + expect(retained.response.source.availability).toBe("available") + const recovered = await manager.refresh(key) + expect(recovered.lastCollectionError).toBeUndefined() + }) +}) diff --git a/desktop/src/main/__tests__/machine-diagnostics-store.test.ts b/desktop/src/main/__tests__/machine-diagnostics-store.test.ts new file mode 100644 index 000000000..05f1773e4 --- /dev/null +++ b/desktop/src/main/__tests__/machine-diagnostics-store.test.ts @@ -0,0 +1,54 @@ +import { mkdtempSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { MachineDiagnosticsStore } from "../machine-diagnostics-store.js" + +const response = (sequence: number) => ({ + schemaVersion: 1, + machine: { id: "machine", context: "default", provider: "test", state: "Running" }, + source: { availability: "available" as const, freshness: "fresh" as const }, + daemon: { sessionId: "session", state: "running" as const, health: "healthy" as const, startedAt: "2026-09-19T00:00:00Z", updatedAt: "2026-09-19T00:00:00Z", workspaceCount: 1 }, + events: [{ schemaVersion: 1, sessionId: "session", sequence, timestamp: `2026-09-19T00:00:0${sequence}Z`, level: "info" as const, type: "daemon.started", message: "started" }], + cursor: { next: `cursor-${sequence}`, state: "ok" as const }, +}) + +describe("MachineDiagnosticsStore", () => { + it("retains the last daemon snapshot and cursor when the remote reader fails", () => { + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + const key = { context: "default", machineId: "machine" } + const good = store.merge(key, response(1)) + const failed = store.merge(key, { ...response(2), daemon: undefined, events: [], source: { availability: "permission_denied", freshness: "unknown", message: "Access denied" }, cursor: { state: "none" } }) + expect(failed.response.daemon).toEqual(good.response.daemon) + expect(failed.cursor).toBe(good.cursor) + expect(failed.lastSuccessfulCollectionAt).toBe(good.lastSuccessfulCollectionAt) + expect(failed.lastCollectionError).toBe("Access denied") + expect(failed.response.source.freshness).toBe("stale") + expect(store.merge(key, response(2)).lastCollectionError).toBeUndefined() + }) + it("deduplicates remote events and retains them after a stop", () => { + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + const key = { context: "default", machineId: "machine" } + store.merge(key, response(1)) + const cache = store.merge(key, response(1)) + expect(cache.events).toHaveLength(1) + expect(cache.cursor).toBe("cursor-1") + expect(store.markStopped(key)?.response.source.availability).toBe("machine_stopped") + expect(store.get(key)?.events).toHaveLength(1) + }) + + it("rejects path traversal in keys", () => { + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + expect(() => store.get({ context: "../other", machineId: "machine" })).toThrow("invalid diagnostics key") + }) + + it("clears a stale cursor after a remote session reset", () => { + const store = new MachineDiagnosticsStore(mkdtempSync(join(tmpdir(), "devsy-diagnostics-"))) + const key = { context: "default", machineId: "machine" } + store.merge(key, response(1)) + const reset = { ...response(2), cursor: { state: "reset" as const, reason: "session_changed" } } + const cache = store.merge(key, reset) + expect(cache.cursor).toBeUndefined() + expect(cache.lastCollectionError).toBeUndefined() + }) +}) diff --git a/desktop/src/main/__tests__/tray.test.ts b/desktop/src/main/__tests__/tray.test.ts index cd3cf30cc..aed9eb07f 100644 --- a/desktop/src/main/__tests__/tray.test.ts +++ b/desktop/src/main/__tests__/tray.test.ts @@ -10,11 +10,25 @@ vi.mock("../updater.js", () => ({ describe("buildUpdateMenuItems", () => { it("returns nothing when no update is downloaded", () => { - expect(buildUpdateMenuItems({ state: "idle", currentVersion: "1.0.0" }, () => {})).toEqual([]) - expect(buildUpdateMenuItems({ state: "checking", currentVersion: "1.0.0" }, () => {})).toEqual([]) expect( buildUpdateMenuItems( - { state: "available", currentVersion: "1.0.0", availableVersion: "1.1.0" }, + { state: "idle", currentVersion: "1.0.0" }, + () => {}, + ), + ).toEqual([]) + expect( + buildUpdateMenuItems( + { state: "checking", currentVersion: "1.0.0" }, + () => {}, + ), + ).toEqual([]) + expect( + buildUpdateMenuItems( + { + state: "available", + currentVersion: "1.0.0", + availableVersion: "1.1.0", + }, () => {}, ), ).toEqual([]) @@ -24,16 +38,36 @@ describe("buildUpdateMenuItems", () => { state: "downloading", currentVersion: "1.0.0", availableVersion: "1.1.0", - progress: { percent: 50, bytesPerSecond: 1000, transferred: 50, total: 100 }, + progress: { + percent: 50, + bytesPerSecond: 1000, + transferred: 50, + total: 100, + }, }, () => {}, ), ).toEqual([]) - expect(buildUpdateMenuItems({ state: "not-available", currentVersion: "1.0.0" }, () => {})).toEqual([]) - expect(buildUpdateMenuItems({ state: "up-to-date", currentVersion: "1.0.0" }, () => {})).toEqual([]) expect( buildUpdateMenuItems( - { state: "error", currentVersion: "1.0.0", error: "x", code: "network" }, + { state: "not-available", currentVersion: "1.0.0" }, + () => {}, + ), + ).toEqual([]) + expect( + buildUpdateMenuItems( + { state: "up-to-date", currentVersion: "1.0.0" }, + () => {}, + ), + ).toEqual([]) + expect( + buildUpdateMenuItems( + { + state: "error", + currentVersion: "1.0.0", + error: "x", + code: "network", + }, () => {}, ), ).toEqual([]) @@ -42,7 +76,11 @@ describe("buildUpdateMenuItems", () => { it("adds Update item + separator when downloaded", () => { const onInstall = vi.fn() const items = buildUpdateMenuItems( - { state: "downloaded", currentVersion: "1.0.0", availableVersion: "9.9.9" }, + { + state: "downloaded", + currentVersion: "1.0.0", + availableVersion: "9.9.9", + }, onInstall, ) expect(items).toHaveLength(2) @@ -122,7 +160,9 @@ describe("buildTrayMenuTemplate", () => { }, actions, ) - expect((empty[2].submenu as Array>)[0]).toMatchObject({ + expect( + (empty[2].submenu as Array>)[0], + ).toMatchObject({ label: "No Active Workspaces", enabled: false, }) @@ -135,8 +175,33 @@ describe("buildTrayMenuTemplate", () => { }, actions, ) - const stop = ((pending[2].submenu as Array>)[0] - .submenu as Array>)[1] + const stop = ( + (pending[2].submenu as Array>)[0] + .submenu as Array> + )[1] expect(stop).toMatchObject({ label: "Stopping…", enabled: false }) }) + it("shows shared progress and permits stopping an active start", () => { + const items = buildTrayMenuTemplate( + { + activeWorkspaces: [{ id: "ws", status: "Stopped" }], + pendingStops: new Set(), + jobs: { + ws: { + commandId: "start", + activity: "starting", + state: "running", + phase: "Building image", + }, + }, + updateStatus: { state: "idle", currentVersion: "1.0.0" }, + }, + actions, + ) + const workspace = (items[2].submenu as Array>)[0] + expect(workspace.label).toBe("ws — Starting") + expect( + (workspace.submenu as Array>)[1], + ).toMatchObject({ label: "Stop Workspace", enabled: true }) + }) }) diff --git a/desktop/src/main/__tests__/watcher.test.ts b/desktop/src/main/__tests__/watcher.test.ts index 695732a38..79513eaff 100644 --- a/desktop/src/main/__tests__/watcher.test.ts +++ b/desktop/src/main/__tests__/watcher.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest" +import { WorkspaceJobs } from "../workspace-jobs.js" import { Watcher } from "../watcher.js" function makeWatcher( @@ -27,7 +28,7 @@ function makeWatcher( runRaw: vi.fn(runRaw), } const providerJobs = { snapshot: vi.fn().mockReturnValue({}) } - const workspaceJobs = { snapshot: vi.fn().mockReturnValue({}) } + const workspaceJobs = new WorkspaceJobs() const watcher = new Watcher({ cli: cli as never, state: state as never, @@ -35,7 +36,7 @@ function makeWatcher( providerJobs: providerJobs as never, workspaceJobs: workspaceJobs as never, }) - return { watcher, cli, state } + return { watcher, cli, state, workspaceJobs } } describe("Watcher.refreshProviders", () => { @@ -193,3 +194,37 @@ describe("Watcher.refreshWorkspaceStatuses", () => { expect(workspaces[1].status).toBe("stopped") }) }) + +describe("workspace status ordering", () => { + it("discards an observation started before an operation", async () => { + let release!: (value: string) => void + const { watcher, state, workspaceJobs } = makeWatcher( + async () => ({}), + [{ id: "ws", status: "Running" }], + () => + new Promise((resolve) => { + release = resolve + }), + ) + const polling = watcher.refreshWorkspaceStatuses() + await Promise.resolve() + workspaceJobs.start("ws", "stopping", "stop") + release('{"state":"Running"}') + await polling + expect(state.updateWorkspaceStatus).not.toHaveBeenCalled() + }) + it("propagates targeted refresh failure without poisoning subsequent refreshes", async () => { + const { watcher, cli } = makeWatcher( + async () => ({}), + [{ id: "ws" }], + async () => { + throw new Error("offline") + }, + ) + await expect(watcher.refreshWorkspaceStatus("ws")).rejects.toThrow( + "offline", + ) + cli.runRaw.mockResolvedValue('{"state":"Stopped"}') + await expect(watcher.refreshWorkspaceStatus("ws")).resolves.toBeUndefined() + }) +}) diff --git a/desktop/src/main/__tests__/workspace-jobs.test.ts b/desktop/src/main/__tests__/workspace-jobs.test.ts index 7ce9caba7..4546e21ae 100644 --- a/desktop/src/main/__tests__/workspace-jobs.test.ts +++ b/desktop/src/main/__tests__/workspace-jobs.test.ts @@ -1,104 +1,101 @@ // @vitest-environment node -import { beforeEach, describe, expect, it, vi } from "vitest" +import { describe, expect, it, vi } from "vitest" import { WorkspaceJobs } from "../workspace-jobs.js" describe("WorkspaceJobs", () => { - let jobs: WorkspaceJobs - - beforeEach(() => { - jobs = new WorkspaceJobs() - }) - - it("tracks a delete until it finishes", async () => { - const generation = jobs.start("ws1") - expect(jobs.get("ws1")).toEqual({ activity: "deleting" }) - - await jobs.finish("ws1", generation) - expect(jobs.get("ws1")).toBeUndefined() - }) - - it("does not let a stale failure land on a newer job", async () => { - const first = jobs.start("ws1") - const second = jobs.start("ws1") - - await jobs.finish("ws1", first, "boom") - - expect(jobs.get("ws1")).toEqual({ activity: "deleting" }) - expect(second).not.toBe(first) - }) - - it("retains the failure so the UI can explain it", async () => { - const generation = jobs.start("ws1") - await jobs.finish("ws1", generation, "delete exited with code 1") - - expect(jobs.get("ws1")).toEqual({ - activity: "deleting", - error: "delete exited with code 1", + it("publishes action state independently of runtime status", async () => { + const jobs = new WorkspaceJobs() + const changed = vi.fn() + jobs.onChange(changed) + const generation = jobs.start("ws", "stopping", "command") + expect(jobs.get("ws")).toMatchObject({ + commandId: "command", + activity: "stopping", + state: "running", }) + jobs.progress("ws", "command", { + phase: "stopping_workspace", + step: "Waiting for lock", + state: "started", + }) + jobs.progress("ws", "command", { + phase: "stopping_workspace", + state: "succeeded", + }) + expect(jobs.get("ws")).toMatchObject({ + state: "running", + phase: "Waiting for lock", + }) + await jobs.finish("ws", generation) + expect(jobs.get("ws")?.state).toBe("succeeded") + expect(changed).toHaveBeenCalled() }) - it("does not let a later release erase a recorded failure", async () => { - const generation = jobs.start("ws1") - await jobs.finish("ws1", generation, "boom") - - await jobs.finish("ws1", generation) - - expect(jobs.get("ws1")?.error).toBe("boom") - }) - - it("ignores a failure for a workspace with no active job", async () => { - await jobs.finish("ws1", 1, "boom") - expect(jobs.get("ws1")).toBeUndefined() + it("rejects duplicates and allows delete to supersede a start", async () => { + const jobs = new WorkspaceJobs() + const first = jobs.start("ws", "starting", "first") + expect(() => jobs.start("ws", "rebuilding", "conflict")).toThrow( + "already in progress", + ) + jobs.start("ws", "deleting", "delete") + jobs.progress("ws", "first", { phase: "ready", state: "succeeded" }) + await jobs.finish("ws", first, "old failure") + expect(jobs.get("ws")).toMatchObject({ + commandId: "delete", + state: "running", + }) }) - it("refreshes the workspace list before clearing a finished job", async () => { - const order: string[] = [] - jobs.setRefresh(async () => { - order.push(`refresh(job=${jobs.get("ws1") ? "present" : "gone"})`) + it("retains a completed delete on refresh failure and retries refresh only", async () => { + const jobs = new WorkspaceJobs() + const refresh = vi + .fn() + .mockRejectedValueOnce(new Error("offline")) + .mockResolvedValue(undefined) + jobs.setRefresh(refresh) + const generation = jobs.start("ws", "deleting", "delete") + await jobs.finish("ws", generation) + expect(jobs.get("ws")).toMatchObject({ + state: "reconciling", + refreshError: "offline", }) - - const generation = jobs.start("ws1") - await jobs.finish("ws1", generation) - - expect(order).toEqual(["refresh(job=present)"]) - expect(jobs.get("ws1")).toBeUndefined() + expect(() => jobs.start("ws")).toThrow() + await jobs.retryRefresh("ws") + expect(jobs.get("ws")).toMatchObject({ + state: "succeeded", + refreshError: undefined, + }) + expect(refresh).toHaveBeenCalledTimes(2) }) - it("does not clear a newer job started while refresh was in flight", async () => { - let releaseRefresh: (() => void) | undefined + it("keeps the action visible until reconciliation finishes and ignores duplicate completion", async () => { + const jobs = new WorkspaceJobs() + let release!: () => void jobs.setRefresh( () => new Promise((resolve) => { - releaseRefresh = resolve + release = resolve }), ) - - const generation = jobs.start("ws1") - const finishing = jobs.finish("ws1", generation) - - jobs.start("ws1") - releaseRefresh?.() + const generation = jobs.start("ws") + const finishing = jobs.finish("ws", generation, "denied") + expect(jobs.get("ws")).toMatchObject({ + state: "reconciling", + error: "denied", + }) + await jobs.finish("ws", generation) + release() await finishing - - expect(jobs.get("ws1")).toEqual({ activity: "deleting" }) - }) - - it("notifies listeners on every mutation", () => { - const listener = vi.fn() - jobs.onChange(listener) - - jobs.start("ws1") - jobs.clear("ws1") - - expect(listener).toHaveBeenCalledTimes(2) + expect(jobs.get("ws")).toMatchObject({ state: "failed", error: "denied" }) + expect(() => jobs.start("ws")).not.toThrow() }) - it("does not notify when clearing an untracked workspace", () => { - const listener = vi.fn() - jobs.onChange(listener) - - jobs.clear("nonexistent") - - expect(listener).not.toHaveBeenCalled() + it("invalidates polls on acceptance and completion", async () => { + const jobs = new WorkspaceJobs() + const before = jobs.generation("ws") + const generation = jobs.start("ws") + expect(generation).not.toBe(before) + await jobs.finish("ws", generation) + expect(jobs.generation("ws")).not.toBe(generation) }) }) diff --git a/desktop/src/main/index.ts b/desktop/src/main/index.ts index b035118b0..f516c25fa 100644 --- a/desktop/src/main/index.ts +++ b/desktop/src/main/index.ts @@ -7,6 +7,8 @@ import { CliRunner } from "./cli.js" import { DaemonManager } from "./daemon-manager.js" import { registerIpcHandlers } from "./ipc.js" import { LogStore } from "./log-store.js" +import { MachineDiagnosticsStore } from "./machine-diagnostics-store.js" +import { MachineDiagnosticsManager } from "./machine-diagnostics-manager.js" import { ProviderJobs } from "./provider-jobs.js" import { PtyManager } from "./pty.js" import { DaemonState } from "./state.js" @@ -156,6 +158,8 @@ app.whenReady().then(() => { // ~/.devsy/contexts//workspaces// subtree that `workspace delete` // unlinks. That separation closes the file-deletion race by construction. const logStore = new LogStore(join(homedir(), ".devsy", "desktop", "logs")) + const machineDiagnosticsStore = new MachineDiagnosticsStore(join(homedir(), ".devsy", "desktop", "diagnostics")) + const machineDiagnosticsManager = new MachineDiagnosticsManager(cli, machineDiagnosticsStore) try { const pruned = logStore.prune(30) if (pruned > 0) console.log(`Pruned ${pruned} old log files`) @@ -204,6 +208,8 @@ app.whenReady().then(() => { cli, state, logStore, + machineDiagnosticsStore, + machineDiagnosticsManager, pty: ptyManager, getMainWindow: () => mainWindow, providerJobs, @@ -222,13 +228,7 @@ app.whenReady().then(() => { pendingRoute = null } }, - onWorkspaceStopComplete: async (workspaceId) => { - try { - await watcher?.refreshWorkspaceStatus(workspaceId) - } finally { - watcher?.broadcastWorkspaces() - } - }, + workspaceSnapshot: () => watcher?.workspaceSnapshot(), }) // Start state watcher @@ -245,9 +245,19 @@ app.whenReady().then(() => { watcher ? watcher.refreshProviders() : Promise.resolve(), ) workspaceJobs.onChange(() => watcher?.broadcastWorkspaces()) - workspaceJobs.setRefresh(() => - watcher ? watcher.refreshWorkspaces() : Promise.resolve(), - ) + workspaceJobs.setRefresh(async (id, job) => { + if (!watcher) throw new Error("Workspace watcher unavailable") + await watcher.refreshWorkspaces() + const exists = state.workspaceList().some((workspace) => workspace.id === id) + if (job.activity === "deleting" && !job.error) { + if (exists) throw new Error("Workspace list has not caught up yet") + } else if (exists) { + await watcher.refreshWorkspaceStatus(id) + } else if (!job.error) { + throw new Error("Workspace not yet present in the list") + } + watcher.broadcastWorkspaces() + }) void watcher.start().then(runInitialProviderUpdateCheck) scheduleProviderUpdateCheck() @@ -255,6 +265,7 @@ app.whenReady().then(() => { // Set up system tray appTray = new AppTray({ state, + workspaceJobs, showDevsy, stopWorkspace: workspaceActions.stop, refreshWorkspace: (id) => diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 1c2a0294f..16a6e2f56 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -15,6 +15,8 @@ import { hashWorkspaceRef, trackEvent } from "./analytics.js" import type { CliRunner } from "./cli.js" import { loadCatalog } from "./image-catalog.js" import type { LogStore } from "./log-store.js" +import type { MachineDiagnosticsStore } from "./machine-diagnostics-store.js" +import type { MachineDiagnosticsManager } from "./machine-diagnostics-manager.js" import type { ProviderActivity, ProviderJobs, @@ -34,6 +36,7 @@ import { } from "./updater.js" import { type ProviderEntry, parseProviderEntries } from "./watcher.js" import { normalizeWorkspaceStatus } from "./workspace-status.js" +import type { WorkspaceActivity } from "../shared/workspace-operation.js" import type { WorkspaceJobs } from "./workspace-jobs.js" const execFileAsync = promisify(execFile) @@ -97,11 +100,13 @@ interface IpcDependencies { cli: CliRunner state: DaemonState logStore: LogStore + machineDiagnosticsStore?: MachineDiagnosticsStore + machineDiagnosticsManager?: MachineDiagnosticsManager pty: PtyManager getMainWindow: () => BrowserWindow | null providerJobs: ProviderJobs workspaceJobs: WorkspaceJobs - onWorkspaceStopComplete?: (workspaceId: string) => Promise + workspaceSnapshot?: () => unknown onRendererReady?: (sender: Electron.WebContents) => void } @@ -191,6 +196,7 @@ function createLogSink( const FLUSH_MS = 64 const MAX_BATCH = 250 let buf: string[] = [] + let finished = false let timer: ReturnType | null = null function post( @@ -209,7 +215,9 @@ function createLogSink( if (!done && buf.length === 0) return const lines = buf buf = [] - getWin()?.webContents.send("command-progress", { + const win = getWin() + if (!win || win.isDestroyed?.()) return + win.webContents.send("command-progress", { commandId, lines: lines.map(redactSensitiveText), done, @@ -225,6 +233,7 @@ function createLogSink( return { line(formatted) { + if (finished) return true const safeLine = redactSensitiveText(formatted) const ok = appendLog?.(safeLine) ?? true buf.push(safeLine) @@ -233,10 +242,17 @@ function createLogSink( return ok }, async done(finalLine, extra) { + if (finished) return + finished = true const safeLine = redactSensitiveText(finalLine) - appendLog?.(safeLine) buf.push(safeLine) - await flush?.() + try { + appendLog?.(safeLine) + await flush?.() + } catch (error) { + // Log persistence must not change a command's actual outcome. + console.warn("[workspace-operation] log flush failed", error) + } post(true, { message: finalLine, ...extra }) }, } @@ -248,7 +264,7 @@ export function registerIpcHandlers(deps: IpcDependencies): { runInitialProviderUpdateCheck: () => void workspaceActions: { stop: (workspaceId: string) => Promise } } { - const { cli, state, logStore, pty, providerJobs, workspaceJobs } = deps + const { cli, state, logStore, pty, providerJobs, workspaceJobs, machineDiagnosticsStore, machineDiagnosticsManager, getMainWindow } = deps const tunnelProcesses = new Map< string, import("node:child_process").ChildProcess @@ -393,36 +409,13 @@ export function registerIpcHandlers(deps: IpcDependencies): { if (stream !== "stdout") return false const envelope = parseCliEnvelope(line) if (envelope?.kind !== "status") return false - deps.getMainWindow()?.webContents.send("workspace-status", { - commandId, - workspaceId, - ...redactOperationStatus(normalizeOperationStatus(envelope)), - }) + if (!workspaceJobs.owns(workspaceId, commandId)) return true + const status = redactOperationStatus(normalizeOperationStatus(envelope)) + workspaceJobs.progress(workspaceId, commandId, status) + deps.getMainWindow()?.webContents.send("workspace-status", { commandId, workspaceId, ...status }) return true } - function emitWorkspaceStatus( - commandId: string, - workspaceId: string, - phase: string, - state: "started" | "succeeded" | "failed", - cliError?: CLIError, - ): void { - deps.getMainWindow()?.webContents.send("workspace-status", { - commandId, - workspaceId, - phase, - state, - ...(cliError - ? { - error: { - ...redactCLIError(cliError), - }, - } - : {}), - }) - } - /** * Track a provider job for the duration of fn, so the job cannot outlive the * work it describes. Opening a job in one place and closing it in another @@ -537,6 +530,10 @@ export function registerIpcHandlers(deps: IpcDependencies): { // ── Workspaces ── ipcMain.handle("workspace_list", () => state.workspaceList()) + ipcMain.handle("workspace_snapshot", () => deps.workspaceSnapshot?.() ?? { + workspaces: state.workspaceList(), jobs: workspaceJobs.snapshot(), revision: workspaceJobs.revision, + }) + ipcMain.handle("workspace_refresh", (_event, args: { workspaceId: string }) => workspaceJobs.retryRefresh(args.workspaceId)) ipcMain.handle( "workspace_status", @@ -551,8 +548,9 @@ export function registerIpcHandlers(deps: IpcDependencies): { "15s", ] if (args.recovery) cliArgs.push("--recovery") + const generation = workspaceJobs.generation(args.workspaceId) const raw = await cli.runRaw(cliArgs) - if (!args.recovery) { + if (!args.recovery && generation === workspaceJobs.generation(args.workspaceId)) { const status = normalizeWorkspaceStatus(raw) if (status && state.updateWorkspaceStatus(args.workspaceId, status)) { // The watcher owns renderer broadcasts; this update still keeps @@ -895,9 +893,11 @@ export function registerIpcHandlers(deps: IpcDependencies): { ipcMain.handle( "machine_delete", async (_event, args: { id: string; force?: boolean }) => { - const cliArgs = ["machine", "delete", args.id] + const key = { context: state.currentContext(), machineId: args.id } + const cliArgs = ["machine", "delete", args.id, "--context", key.context] if (args.force) cliArgs.push("--force") await cli.runRaw(cliArgs) + machineDiagnosticsManager?.delete(key) }, ) @@ -906,13 +906,27 @@ export function registerIpcHandlers(deps: IpcDependencies): { }) ipcMain.handle("machine_stop", async (_event, args: { id: string }) => { - await cli.runRaw(["machine", "stop", args.id]) + const key = { context: state.currentContext(), machineId: args.id } + await cli.runRaw(["machine", "stop", args.id, "--context", key.context]) + machineDiagnosticsManager?.markStopped(key) }) ipcMain.handle("machine_status", async (_event, args: { id: string }) => { return cli.runRaw(["machine", "status", args.id, "--result-format", "json"]) }) + ipcMain.handle("machine_diagnostics_get", (_event, args: { id: string }) => { + return machineDiagnosticsManager?.getCached({ context: state.currentContext(), machineId: args.id }) ?? null + }) + + ipcMain.handle("machine_diagnostics_refresh", async (_event, args: { id: string }) => { + if (!machineDiagnosticsManager) throw new Error("machine diagnostics manager is unavailable") + const key = { context: state.currentContext(), machineId: args.id } + const merged = await machineDiagnosticsManager.refresh(key) + getMainWindow()?.webContents.send("machine-diagnostics-changed", { machineId: args.id, context: key.context, diagnostics: merged }) + return merged + }) + // ── Contexts ── ipcMain.handle("context_list", () => state.contextList()) @@ -1116,160 +1130,288 @@ export function registerIpcHandlers(deps: IpcDependencies): { const wsId = args.workspaceId ?? args.source const cmdId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile(state.workspaceContext(wsId), wsId) - const sink = createLogSink( - deps.getMainWindow, - cmdId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), - ) + const activity = args.workspaceId ? "creating" : "starting" + const generation = workspaceJobs.start(wsId, activity, cmdId) + const started = performance.now() + const timing = (stage: string) => + console.debug( + `[workspace-operation] ${cmdId} ${stage} ${Math.round(performance.now() - started)}ms`, + ) + timing("accepted") + void (async () => { + const logPath = logStore.createLogFile( + state.workspaceContext(wsId), + wsId, + ) + const sink = createLogSink( + deps.getMainWindow, + cmdId, + (line) => logStore.appendLog(logPath, line), + () => logStore.closeLog(logPath), + ) - return serializePerWorkspace(wsId, async () => { - // Tear down any prior run for this workspace before starting a new - // one. - let taskId: string - try { - await cancelActiveUp(wsId) - // Submit: returns immediately with the background task's id. - const submitted = await cli.run<{ kind: string; id: string }>([ - ...cliArgs, - "--detach", - ]) - if (!submitted?.id) { - throw new Error("workspace up --detach returned no task id") + await serializePerWorkspace(wsId, async () => { + // Tear down any prior run for this workspace before starting a new + // one. + let taskId: string + try { + workspaceJobs.phase(wsId, cmdId, "Closing connections") + await cancelActiveUp(wsId) + timing("shutdown-complete") + workspaceJobs.phase(wsId, cmdId, "Launching command") + timing("cli-launch") + // Submit: returns immediately with the background task's id. + const submitted = await cli.run<{ kind: string; id: string }>([ + ...cliArgs, + "--detach", + ]) + if (!submitted?.id) { + throw new Error("workspace up --detach returned no task id") + } + taskId = submitted.id + } catch (error) { + const err = error as Error & { cliError?: CLIError } + void sink.done(formatLogLine(err.message, "ERROR"), { + level: "error", + success: false, + cliError: err.cliError ?? { + code: "up_failed", + message: err.message, + }, + }) + await workspaceJobs.finish( + wsId, + generation, + redactCLIError( + err.cliError ?? { code: "up_failed", message: err.message }, + ).message, + ) + return cmdId } - taskId = submitted.id - } catch (error) { - const err = error as Error & { cliError?: CLIError } - void sink.done(formatLogLine(err.message, "ERROR"), { - level: "error", - success: false, - cliError: err.cliError ?? { - code: "up_failed", - message: err.message, - }, - }) - return cmdId - } - activeUpTasks.set(wsId, taskId) + activeUpTasks.set(wsId, taskId) - // A newer submission may already own the entry and must stay cancellable. - const releaseTask = () => { - if (activeUpTasks.get(wsId) === taskId) { - activeUpTasks.delete(wsId) + // A newer submission may already own the entry and must stay cancellable. + const releaseTask = () => { + if (activeUpTasks.get(wsId) === taskId) { + activeUpTasks.delete(wsId) + } } - } - - let signalledDone = false - let suppressCallbacks = false - const sendWorkspaceFailureStatus = (cliError: CLIError | undefined) => { - const safeError = redactCLIError( - cliError ?? { code: "up_failed", message: "workspace up failed" }, - ) - deps.getMainWindow()?.webContents.send("workspace-status", { - commandId: cmdId, - workspaceId: wsId, - phase: "failed", - state: "failed", - error: safeError, - }) - } - let child: import("node:child_process").ChildProcess - try { - child = await cli.runStreaming( - ["workspace", "task", "logs", taskId, "--follow"], - (line, stream) => { - if (signalledDone || suppressCallbacks) return - - // Structured NDJSON envelopes only ever appear on stdout; stderr - // carries freeform zap log lines. - const envelope = - stream === "stdout" ? parseCliEnvelope(line) : undefined - - if (envelope?.kind === "status") { - const status = redactOperationStatus(normalizeOperationStatus(envelope)) - deps.getMainWindow()?.webContents.send("workspace-status", { - commandId: cmdId, - workspaceId: wsId, - ...status, - }) - return - } - - const formatted = formatLogLine(line) - - if (envelope?.kind === "result") { - signalledDone = true - releaseTask() - void sink.done(formatted, { success: true }) - return - } - - if (envelope?.kind === "error") { - signalledDone = true - releaseTask() - void sink.done(formatted, { - level: "error", - success: false, - cliError: { - code: envelope.code ?? "up_failed", - message: envelope.message, - hint: envelope.hint, - context: envelope.context, - }, - }) - return - } - if (!sink.line(formatted)) return logStore.onDrain(logPath) - }, - (code, cliError) => { - // No releaseTask: the follower dying says nothing about the - // detached worker, and would orphan a still-running task. - if (tunnelProcesses.get(wsId) === child) { - tunnelProcesses.delete(wsId) - } - if (signalledDone || suppressCallbacks) return - if (code !== 0) sendWorkspaceFailureStatus(cliError) - void sink.done( - formatLogLine( - `Exit code: ${code}`, - code === 0 ? "INFO" : "ERROR", - ), - code === 0 - ? { success: true } - : { level: "error", success: false, cliError }, - ) - }, - wsId, - ) - // Expose a method to suppress callbacks from cancelActiveUp - ;(child as unknown as { _suppressWorkspaceCallbacks?: () => void })._suppressWorkspaceCallbacks = () => { - suppressCallbacks = true + let firstProgress = true + let signalledDone = false + let suppressCallbacks = false + let child: import("node:child_process").ChildProcess + try { + child = await cli.runStreaming( + ["workspace", "task", "logs", taskId, "--follow"], + (line, stream) => { + if ( + signalledDone || + suppressCallbacks || + !workspaceJobs.owns(wsId, cmdId) + ) + return + + if (firstProgress) { + timing("first-progress") + firstProgress = false + } + // Structured NDJSON envelopes only ever appear on stdout; stderr + // carries freeform zap log lines. + const envelope = + stream === "stdout" ? parseCliEnvelope(line) : undefined + + if (envelope?.kind === "status") { + const status = redactOperationStatus( + normalizeOperationStatus(envelope), + ) + workspaceJobs.progress(wsId, cmdId, status) + deps.getMainWindow()?.webContents.send("workspace-status", { + commandId: cmdId, + workspaceId: wsId, + ...status, + }) + return + } + + const formatted = formatLogLine(line) + + if (envelope?.kind === "result") { + signalledDone = true + releaseTask() + timing("completed") + void workspaceJobs.finish(wsId, generation) + void sink.done(formatted, { success: true }) + return + } + + if (envelope?.kind === "error") { + signalledDone = true + releaseTask() + void workspaceJobs.finish( + wsId, + generation, + redactCLIError({ + code: envelope.code ?? "up_failed", + message: envelope.message, + }).message, + ) + void sink.done(formatted, { + level: "error", + success: false, + cliError: { + code: envelope.code ?? "up_failed", + message: envelope.message, + hint: envelope.hint, + context: envelope.context, + }, + }) + return + } + + if (!sink.line(formatted)) return logStore.onDrain(logPath) + }, + (code, cliError) => { + // No releaseTask: the follower dying says nothing about the + // detached worker, and would orphan a still-running task. + if (tunnelProcesses.get(wsId) === child) { + tunnelProcesses.delete(wsId) + } + if ( + signalledDone || + suppressCallbacks || + !workspaceJobs.owns(wsId, cmdId) + ) + return + // A follower exiting is not evidence that the detached task finished. + void reconcileDetachedTask( + wsId, + cmdId, + generation, + taskId, + sink, + releaseTask, + ) + }, + wsId, + ) + // Expose a method to suppress callbacks from cancelActiveUp + ;( + child as unknown as { _suppressWorkspaceCallbacks?: () => void } + )._suppressWorkspaceCallbacks = () => { + suppressCallbacks = true + } + } catch (error) { + // A failed follower does not prove that the detached task failed. + // Keep its cancellation handle and reconcile from persisted task state. + void reconcileDetachedTask( + wsId, + cmdId, + generation, + taskId, + sink, + releaseTask, + ) + return cmdId } - } catch (error) { - // The task is already submitted; keep it registered so a later - // cancel can still reach it, and close the sink so the UI isn't - // left waiting on a follower that never started. - const err = error as Error & { cliError?: CLIError } - sendWorkspaceFailureStatus(err.cliError) - void sink.done(formatLogLine(err.message, "ERROR"), { - level: "error", - success: false, - cliError: err.cliError ?? { - code: "up_follow_failed", - message: err.message, - }, - }) - return cmdId - } - tunnelProcesses.set(wsId, child) + tunnelProcesses.set(wsId, child) - return cmdId + return cmdId + }) + })().catch(async (error) => { + await workspaceJobs.finish( + wsId, + generation, + redactCLIError(cliErrorOrFallback(error, "up_failed")).message, + ) }) + return cmdId }, ) + async function reconcileDetachedTask( + wsId: string, + commandId: string, + generation: number, + taskId: string, + sink: ProgressSink, + release: () => void, + ): Promise { + while ( + workspaceJobs.owns(wsId, commandId) && + workspaceJobs.get(wsId)?.state === "running" + ) { + try { + const raw = await cli.runRaw([ + "workspace", + "task", + "get", + taskId, + "--result-format", + "json", + ]) + if (!workspaceJobs.owns(wsId, commandId)) return + for (const line of raw.split("\n")) { + const envelope = parseCliEnvelope(line) + if (envelope?.kind === "result") { + release() + await sink + .done(formatLogLine("Completed"), { success: true }) + .catch((error) => + console.warn("[workspace-operation] log flush failed", error), + ) + await workspaceJobs.finish(wsId, generation) + return + } + if (envelope?.kind === "status") + workspaceJobs.progress( + wsId, + commandId, + redactOperationStatus(normalizeOperationStatus(envelope)), + ) + } + } catch (error) { + // task get can also fail because the store/CLI is unavailable. Only a + // persisted terminal task state authorizes declaring the worker failed. + try { + const tasks = await cli.run< + Array<{ + id: string + status: string + error?: string + errorCode?: string + }> + >(["workspace", "task", "list"]) + const task = tasks.find((task) => task.id === taskId) + if ( + task?.status === "failed" && + workspaceJobs.owns(wsId, commandId) + ) { + release() + const safe = redactCLIError({ + code: task.errorCode ?? "up_failed", + message: task.error || "Workspace task failed", + }) + await sink.done(formatLogLine(safe.message, "ERROR"), { + success: false, + cliError: safe, + }) + await workspaceJobs.finish(wsId, generation, safe.message) + return + } + } catch { + /* Keep the operation and cancellation handle while unavailable. */ + } + workspaceJobs.phase( + wsId, + commandId, + "Progress unavailable · Checking task", + ) + } + await new Promise((resolve) => setTimeout(resolve, 3000)) + } + } + type WorkspaceActionSource = "renderer" | "tray" type StopWorkspaceArgs = { workspaceId: string @@ -1277,337 +1419,164 @@ export function registerIpcHandlers(deps: IpcDependencies): { commandId?: string } - async function startWorkspaceStop( + function startWorkspaceAction( args: StopWorkspaceArgs, - source: WorkspaceActionSource, - ): Promise<{ commandId: string; completion: Promise }> { - trackEvent("workspace_stop", { - workspace_ref: hashWorkspaceRef(args.workspaceId), - source, - }) - await quiesceWorkspace(args.workspaceId) + activity: WorkspaceActivity, + cliArgs: string[], + source: WorkspaceActionSource = "renderer", + ): { commandId: string; completion: Promise } { const commandId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile( - state.workspaceContext(args.workspaceId), + const generation = workspaceJobs.start( args.workspaceId, - ) - const sink = createLogSink( - deps.getMainWindow, + activity, commandId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), ) - const completion = new Promise((resolve, reject) => { - const cliArgs = ["workspace", "stop", args.workspaceId] - if (args.debug) cliArgs.push("--debug") - emitWorkspaceStatus(commandId, args.workspaceId, "stopping_workspace", "started") - void cli.runStreaming( - cliArgs, - (line, stream) => { - if (forwardWorkspaceStatus(commandId, args.workspaceId, line, stream)) return - if (!sink.line(formatLogLine(line))) return logStore.onDrain(logPath) - }, - (code, cliError) => { - if (code === 0) { - emitWorkspaceStatus(commandId, args.workspaceId, "stopping_workspace", "succeeded") - } else { - emitWorkspaceStatus( - commandId, - args.workspaceId, - "stopping_workspace", - "failed", - cliError ?? { - code: "workspace_stop_failed", - message: `workspace stop exited with code ${code}`, + const started = performance.now() + const timing = (stage: string) => + console.debug( + `[workspace-operation] ${commandId} ${stage} ${Math.round(performance.now() - started)}ms`, + ) + trackEvent( + `workspace_${{ stopping: "stop", deleting: "delete", rebuilding: "rebuild", resetting: "reset", starting: "up", creating: "create" }[activity]}`, + { workspace_ref: hashWorkspaceRef(args.workspaceId), source }, + ) + timing("accepted") + const completion = (async () => { + let sink: ProgressSink | undefined + try { + workspaceJobs.phase(args.workspaceId, commandId, "Closing connections") + await quiesceWorkspace(args.workspaceId) + timing("shutdown-complete") + const logPath = logStore.createLogFile( + state.workspaceContext(args.workspaceId), + args.workspaceId, + ) + sink = createLogSink( + deps.getMainWindow, + commandId, + (line) => logStore.appendLog(logPath, line), + () => logStore.closeLog(logPath), + ) + if (args.debug) cliArgs.push("--debug") + workspaceJobs.phase(args.workspaceId, commandId, "Launching command") + timing("cli-launch") + let firstProgress = true + await new Promise((resolve, reject) => { + void cli + .runStreaming( + cliArgs, + (line, stream) => { + if (!workspaceJobs.owns(args.workspaceId, commandId)) return + if (firstProgress) { + timing("first-progress") + firstProgress = false + } + if ( + forwardWorkspaceStatus( + commandId, + args.workspaceId, + line, + stream, + ) + ) + return + if (!sink!.line(formatLogLine(line))) + return logStore.onDrain(logPath) }, + (code, cliError) => { + if (code === 0) resolve() + else + reject( + Object.assign( + new Error( + cliError?.message ?? `Command exited with code ${code}`, + ), + { cliError }, + ), + ) + }, + args.workspaceId, ) - } - void sink - .done( - formatLogLine(`Exit code: ${code}`, code === 0 ? "INFO" : "ERROR"), - { success: code === 0 }, - ) - .then(() => { - if (code === 0) resolve() - else reject(new Error(`workspace stop exited with code ${code}`)) - }) .catch(reject) - }, - args.workspaceId, - ).catch((error) => { - const cliError = cliErrorOrFallback(error, "workspace_stop_failed") - emitWorkspaceStatus( - commandId, - args.workspaceId, - "stopping_workspace", - "failed", - cliError, + }) + timing("completed") + await sink + .done(formatLogLine("Completed"), { success: true }) + .catch((error) => + console.warn("[workspace-operation] log flush failed", error), + ) + await workspaceJobs.finish(args.workspaceId, generation) + timing("reconciled") + } catch (error) { + const cliError = redactCLIError( + cliErrorOrFallback(error, "workspace_operation_failed"), ) - void sink - .done(formatLogLine(cliError.message, "ERROR"), { - level: "error", - success: false, - cliError, - }) - .catch(() => {}) - .finally(() => reject(error)) - }) - }) - void completion - .then( - () => deps.onWorkspaceStopComplete?.(args.workspaceId), - () => deps.onWorkspaceStopComplete?.(args.workspaceId), - ) - .catch((error) => { - console.warn( - `[ipc] failed to reconcile workspace ${args.workspaceId}:`, - error, + if (sink) + await sink + .done(formatLogLine(cliError.message, "ERROR"), { + success: false, + level: "error", + cliError, + }) + .catch(() => {}) + await workspaceJobs.finish( + args.workspaceId, + generation, + cliError.message, ) - }) + throw error + } + })() + // IPC acknowledges acceptance. All subsequent errors live in the shared job. + void completion.catch(() => {}) return { commandId, completion } } - ipcMain.handle("workspace_stop", async (_event, args: StopWorkspaceArgs) => { - const { commandId, completion } = await startWorkspaceStop(args, "renderer") - void completion.catch(() => {}) - return commandId - }) - + function startWorkspaceStop( + args: StopWorkspaceArgs, + source: WorkspaceActionSource, + ) { + return startWorkspaceAction( + args, + "stopping", + ["workspace", "stop", args.workspaceId], + source, + ) + } + ipcMain.handle( + "workspace_stop", + (_event, args: StopWorkspaceArgs) => + startWorkspaceStop(args, "renderer").commandId, + ) ipcMain.handle( "workspace_delete", - async (_event, args: { workspaceId: string; debug?: boolean; commandId?: string }) => { - trackEvent("workspace_delete", { - workspace_ref: hashWorkspaceRef(args.workspaceId), - }) - // Replaces the old `devsy down` command which the CLI overhaul removed: - // before invoking delete, terminate every desktop-spawned child tied to - // this workspace and wait for them to actually exit. Otherwise late - // stdout/stderr lands on a log file the CLI is about to unlink, causing - // an ENOENT crash in the main process. - await quiesceWorkspace(args.workspaceId) - const cmdId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile( - state.workspaceContext(args.workspaceId), - args.workspaceId, - ) - const sink = createLogSink( - deps.getMainWindow, - cmdId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), - ) - - const cliArgs = ["workspace", "delete", args.workspaceId] - if (args.debug) cliArgs.push("--debug") - cliArgs.push("--force") - - // The card shows "Deleting" until finish() below - const jobGeneration = workspaceJobs.start(args.workspaceId) - emitWorkspaceStatus(cmdId, args.workspaceId, "deleting_workspace", "started") - - void cli.runStreaming( - cliArgs, - (line, stream) => { - if (forwardWorkspaceStatus(cmdId, args.workspaceId, line, stream)) return - if (!sink.line(formatLogLine(line))) return logStore.onDrain(logPath) - }, - (code, cliError) => { - if (code === 0) { - emitWorkspaceStatus(cmdId, args.workspaceId, "deleting_workspace", "succeeded") - } else { - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "deleting_workspace", - "failed", - cliError ?? { - code: "workspace_delete_failed", - message: `workspace delete exited with code ${code}`, - }, - ) - } - void sink.done( - formatLogLine(`Exit code: ${code}`, code === 0 ? "INFO" : "ERROR"), - { success: code === 0 }, - ) - void workspaceJobs.finish( - args.workspaceId, - jobGeneration, - code === 0 - ? undefined - : cliError?.message ?? `delete exited with code ${code}`, - ) - }, + (_event, args: StopWorkspaceArgs) => + startWorkspaceAction(args, "deleting", [ + "workspace", + "delete", args.workspaceId, - ).catch((error) => { - const cliError = cliErrorOrFallback(error, "workspace_delete_failed") - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "deleting_workspace", - "failed", - cliError, - ) - void sink.done(formatLogLine(cliError.message, "ERROR"), { - level: "error", - success: false, - cliError, - }) - void workspaceJobs.finish(args.workspaceId, jobGeneration, cliError.message) - }) - - return cmdId - }, + "--force", + ]).commandId, ) - ipcMain.handle( "workspace_rebuild", - async (_event, args: { workspaceId: string; debug?: boolean; commandId?: string }) => { - trackEvent("workspace_rebuild", { - workspace_ref: hashWorkspaceRef(args.workspaceId), - }) - const cmdId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile( - state.workspaceContext(args.workspaceId), - args.workspaceId, - ) - const sink = createLogSink( - deps.getMainWindow, - cmdId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), - ) - - const cliArgs = ["workspace", "up", args.workspaceId, "--recreate"] - if (args.debug) cliArgs.push("--debug") - let sawFailedStatus = false - - void cli.runStreaming( - cliArgs, - (line, stream) => { - const envelope = stream === "stdout" ? parseCliEnvelope(line) : undefined - if (envelope?.kind === "status" && envelope.state === "failed") { - sawFailedStatus = true - } - if (forwardWorkspaceStatus(cmdId, args.workspaceId, line, stream)) { - return - } - if (!sink.line(formatLogLine(line))) return logStore.onDrain(logPath) - }, - (code, cliError) => { - if (code !== 0 && !sawFailedStatus) { - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "rebuilding_workspace", - "failed", - cliError ?? { - code: "workspace_rebuild_failed", - message: `workspace rebuild exited with code ${code}`, - }, - ) - } - void sink.done( - formatLogLine(`Exit code: ${code}`, code === 0 ? "INFO" : "ERROR"), - code === 0 - ? { success: true } - : { level: "error", success: false, cliError }, - ) - }, + (_event, args: StopWorkspaceArgs) => + startWorkspaceAction(args, "rebuilding", [ + "workspace", + "up", args.workspaceId, - ).catch((error) => { - const cliError = cliErrorOrFallback(error, "workspace_rebuild_failed") - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "rebuilding_workspace", - "failed", - cliError, - ) - void sink.done(formatLogLine(cliError.message, "ERROR"), { - level: "error", - success: false, - cliError, - }) - }) - - return cmdId - }, + "--recreate", + ]).commandId, ) - ipcMain.handle( "workspace_reset", - async (_event, args: { workspaceId: string; debug?: boolean; commandId?: string }) => { - trackEvent("workspace_reset", { - workspace_ref: hashWorkspaceRef(args.workspaceId), - }) - const cmdId = args.commandId ?? crypto.randomUUID() - const logPath = logStore.createLogFile( - state.workspaceContext(args.workspaceId), - args.workspaceId, - ) - const sink = createLogSink( - deps.getMainWindow, - cmdId, - (line) => logStore.appendLog(logPath, line), - () => logStore.closeLog(logPath), - ) - - const cliArgs = ["workspace", "up", args.workspaceId, "--reset"] - if (args.debug) cliArgs.push("--debug") - let sawFailedStatus = false - - void cli.runStreaming( - cliArgs, - (line, stream) => { - const envelope = stream === "stdout" ? parseCliEnvelope(line) : undefined - if (envelope?.kind === "status" && envelope.state === "failed") { - sawFailedStatus = true - } - if (forwardWorkspaceStatus(cmdId, args.workspaceId, line, stream)) { - return - } - if (!sink.line(formatLogLine(line))) return logStore.onDrain(logPath) - }, - (code, cliError) => { - if (code !== 0 && !sawFailedStatus) { - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "resetting_workspace", - "failed", - cliError ?? { - code: "workspace_reset_failed", - message: `workspace reset exited with code ${code}`, - }, - ) - } - void sink.done( - formatLogLine(`Exit code: ${code}`, code === 0 ? "INFO" : "ERROR"), - code === 0 - ? { success: true } - : { level: "error", success: false, cliError }, - ) - }, + (_event, args: StopWorkspaceArgs) => + startWorkspaceAction(args, "resetting", [ + "workspace", + "up", args.workspaceId, - ).catch((error) => { - const cliError = cliErrorOrFallback(error, "workspace_reset_failed") - emitWorkspaceStatus( - cmdId, - args.workspaceId, - "resetting_workspace", - "failed", - cliError, - ) - void sink.done(formatLogLine(cliError.message, "ERROR"), { - level: "error", - success: false, - cliError, - }) - }) - - return cmdId - }, + "--reset", + ]).commandId, ) // ── Terminal ── diff --git a/desktop/src/main/machine-diagnostics-manager.ts b/desktop/src/main/machine-diagnostics-manager.ts new file mode 100644 index 000000000..d9982d441 --- /dev/null +++ b/desktop/src/main/machine-diagnostics-manager.ts @@ -0,0 +1,66 @@ +import type { MachineDiagnosticsCache, MachineDiagnosticsResponse } from "../shared/machine-diagnostics-types.js" +import type { CliRunner } from "./cli.js" +import { MachineDiagnosticsStore, type MachineDiagnosticsKey } from "./machine-diagnostics-store.js" + +function keyOf(key: MachineDiagnosticsKey): string { + return JSON.stringify([key.context, key.machineId]) +} + +export class MachineDiagnosticsManager { + private readonly inFlight = new Map>() + private readonly invalidated = new Set() + + constructor( + private readonly cli: CliRunner, + private readonly store: MachineDiagnosticsStore, + ) {} + + getCached(key: MachineDiagnosticsKey): MachineDiagnosticsCache | null { + return this.store.get(key) + } + + refresh(key: MachineDiagnosticsKey): Promise { + const id = keyOf(key) + const active = this.inFlight.get(id) + if (active) return active + this.invalidated.delete(id) + + const request = this.collect(key).finally(() => { + this.inFlight.delete(id) + this.invalidated.delete(id) + }) + this.inFlight.set(id, request) + return request + } + + markStopped(key: MachineDiagnosticsKey): MachineDiagnosticsCache | null { + if (this.inFlight.has(keyOf(key))) this.invalidated.add(keyOf(key)) + return this.store.markStopped(key) + } + + delete(key: MachineDiagnosticsKey): void { + if (this.inFlight.has(keyOf(key))) this.invalidated.add(keyOf(key)) + this.store.delete(key) + } + + private async collect(key: MachineDiagnosticsKey): Promise { + const cached = this.store.get(key) + const args = ["machine", "diagnostics", key.machineId, "--context", key.context, "--result-format", "json", "--limit", "200"] + if (cached?.cursor) args.push("--after", cached.cursor) + + try { + const response = JSON.parse(await this.cli.runRaw(args)) as MachineDiagnosticsResponse + if (this.invalidated.has(keyOf(key))) throw new Error("Diagnostics collection superseded by a machine lifecycle change.") + if (response.schemaVersion !== 1 || response.machine?.id !== key.machineId || response.machine?.context !== key.context || !response.source || !response.cursor) { + throw new Error("Remote diagnostics returned an invalid or mismatched response.") + } + return this.store.merge(key, response) + } catch (error) { + if (this.invalidated.has(keyOf(key))) throw error + const message = error instanceof Error ? error.message : "Remote diagnostics collection failed." + const retained = this.store.recordFailure(key, message) + if (retained) return retained + throw error + } + } +} diff --git a/desktop/src/main/machine-diagnostics-store.ts b/desktop/src/main/machine-diagnostics-store.ts new file mode 100644 index 000000000..9dc17d1f7 --- /dev/null +++ b/desktop/src/main/machine-diagnostics-store.ts @@ -0,0 +1,100 @@ +import { randomUUID } from "node:crypto" +import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs" +import { basename, join } from "node:path" +import type { MachineDiagnosticEvent, MachineDiagnosticsCache, MachineDiagnosticsResponse } from "../shared/machine-diagnostics-types.js" + +export interface MachineDiagnosticsKey { + context: string + machineId: string +} + +function safe(value: string): string { + const clean = basename(value) + if (!clean || clean === "." || clean === ".." || clean !== value) throw new Error("invalid diagnostics key") + return clean +} + +function eventKey(event: MachineDiagnosticEvent): string { + return `${event.sessionId}:${event.sequence}` +} + +export class MachineDiagnosticsStore { + constructor(private readonly root: string) {} + private path(key: MachineDiagnosticsKey): string { + return join(this.root, "machines", safe(key.context), `${safe(key.machineId)}.json`) + } + + get(key: MachineDiagnosticsKey): MachineDiagnosticsCache | null { + const path = this.path(key) + try { + const cache = JSON.parse(readFileSync(path, "utf-8")) as MachineDiagnosticsCache + if (cache.response?.schemaVersion !== 1 || !Array.isArray(cache.events)) return null + return cache + } catch { + return null + } + } + merge(key: MachineDiagnosticsKey, response: MachineDiagnosticsResponse): MachineDiagnosticsCache { + if (response.source.availability !== "available") { + const retained = this.get(key) + if (retained) { + retained.response.source = { ...response.source, freshness: "stale" } + retained.response.machine = response.machine + retained.lastAttemptAt = new Date().toISOString() + retained.lastCollectionError = response.source.availability === "machine_stopped" ? undefined : (response.source.message ?? `Diagnostics are ${response.source.availability}.`) + this.write(key, retained) + return retained + } + } + const existing = this.get(key) + const events = [...(existing?.events ?? []), ...(response.events ?? [])] + const unique = new Map(events.map(event => [eventKey(event), event])) + const merged = [...unique.values()].sort((a, b) => a.timestamp.localeCompare(b.timestamp) || a.sequence - b.sequence).slice(-2000) + const { events: _events, ...responseWithoutEvents } = response + const now = new Date().toISOString() + const cache: MachineDiagnosticsCache = { + response: responseWithoutEvents, + events: merged, + cursor: response.cursor.state === "reset" ? undefined : (response.cursor.next ?? existing?.cursor), + lastAttemptAt: now, + lastSuccessfulCollectionAt: response.source.availability === "available" ? now : existing?.lastSuccessfulCollectionAt, + historyGap: Boolean(existing?.historyGap || response.cursor.state === "gap"), + } + this.write(key, cache) + return cache + } + markStopped(key: MachineDiagnosticsKey): MachineDiagnosticsCache | null { + const cache = this.get(key) + if (!cache) return null + cache.response.source = { availability: "machine_stopped", freshness: "unknown" } + cache.lastCollectionError = undefined + this.write(key, cache) + return cache + } + + recordFailure(key: MachineDiagnosticsKey, message: string): MachineDiagnosticsCache | null { + const cache = this.get(key) + if (!cache) return null + cache.lastAttemptAt = new Date().toISOString() + cache.lastCollectionError = message + cache.response.source.freshness = "stale" + this.write(key, cache) + return cache + } + + delete(key: MachineDiagnosticsKey): void { + const path = this.path(key) + if (existsSync(path)) unlinkSync(path) + } + private write(key: MachineDiagnosticsKey, cache: MachineDiagnosticsCache): void { + const path = this.path(key) + mkdirSync(join(this.root, "machines", safe(key.context)), { recursive: true, mode: 0o700 }) + const tmp = `${path}.${randomUUID()}.tmp` + try { + writeFileSync(tmp, JSON.stringify(cache), { mode: 0o600 }) + renameSync(tmp, path) + } finally { + if (existsSync(tmp)) unlinkSync(tmp) + } + } +} diff --git a/desktop/src/main/tray.ts b/desktop/src/main/tray.ts index 61e11123e..67471dfb5 100644 --- a/desktop/src/main/tray.ts +++ b/desktop/src/main/tray.ts @@ -1,3 +1,10 @@ +import { + workspaceJobBusy, + workspaceJobInterruptible, + workspaceJobLabel, + type WorkspaceJob, +} from "../shared/workspace-operation.js" +import type { WorkspaceJobs } from "./workspace-jobs.js" import { join } from "node:path" import { app, Menu, nativeImage, nativeTheme, Tray } from "electron" import type { DaemonState, Workspace } from "./state.js" @@ -22,14 +29,12 @@ export function buildUpdateMenuItems( : version ? `Update to ${version}` : "Restart" - return [ - { label, click: onInstall }, - { type: "separator" }, - ] + return [{ label, click: onInstall }, { type: "separator" }] } export interface TrayMenuModel { activeWorkspaces: Workspace[] + jobs?: Record pendingStops: ReadonlySet updateStatus: UpdateStatus } @@ -51,19 +56,25 @@ export function buildTrayMenuTemplate( const workspaceItems: Electron.MenuItemConstructorOptions[] = active .slice(0, 10) .map((workspace) => { - const pending = model.pendingStops.has(workspace.id) + const job = model.jobs?.[workspace.id] + const pending = + model.pendingStops.has(workspace.id) || workspaceJobBusy(job) const busy = workspace.status?.trim().toLowerCase() === "busy" + const disabled = pending && !workspaceJobInterruptible(job) + const label = workspaceJobLabel(job) return { - label: `${workspace.id}${busy && !pending ? " — Busy" : ""}`, + label: `${workspace.id}${label ? ` — ${label}` : busy && !pending ? " — Busy" : ""}`, submenu: [ { label: "Open in Devsy", click: () => actions.showWorkspace(workspace.id), }, { - label: pending ? "Stopping…" : "Stop Workspace", - enabled: !pending, - click: pending + label: disabled + ? `${workspaceJobLabel(job) ?? "Stopping"}…` + : "Stop Workspace", + enabled: !disabled, + click: disabled ? undefined : () => actions.stopWorkspace(workspace.id), }, @@ -101,6 +112,7 @@ export function buildTrayMenuTemplate( } interface TrayDeps { + workspaceJobs?: WorkspaceJobs state: DaemonState showDevsy: (route?: string) => void stopWorkspace: (workspaceId: string) => Promise @@ -112,6 +124,7 @@ export class AppTray { private tray: Tray | null = null private pendingStops = new Set() private unsubscribeWorkspaceState: (() => void) | null = null + private unsubscribeWorkspaceJobs: (() => void) | null = null private unsubscribeUpdateStatus: (() => void) | null = null private readonly onThemeUpdated = (): void => { this.tray?.setImage(this.createTrayIcon()) @@ -126,6 +139,8 @@ export class AppTray { this.unsubscribeWorkspaceState = this.deps.state.onWorkspacesChange(() => this.rebuildMenu(), ) + this.unsubscribeWorkspaceJobs = + this.deps.workspaceJobs?.onChange(() => this.rebuildMenu()) ?? null this.unsubscribeUpdateStatus = onUpdateStatusChanged(() => this.rebuildMenu(), ) @@ -136,6 +151,8 @@ export class AppTray { } destroy(): void { + this.unsubscribeWorkspaceJobs?.() + this.unsubscribeWorkspaceJobs = null this.unsubscribeWorkspaceState?.() this.unsubscribeWorkspaceState = null this.unsubscribeUpdateStatus?.() @@ -158,21 +175,31 @@ export class AppTray { return icon } const variant = nativeTheme.shouldUseDarkColors ? "dark" : "light" - return nativeImage.createFromPath( - join(trayDir, `icon-tray-${variant}.png`), - ) + return nativeImage.createFromPath(join(trayDir, `icon-tray-${variant}.png`)) } private rebuildMenu(): void { if (!this.tray) return - const activeWorkspaces = this.deps.state - .workspaceList() - .filter((workspace) => isActiveWorkspaceStatus(workspace.status)) + const jobs = this.deps.workspaceJobs?.snapshot() ?? {} + const workspaces = this.deps.state.workspaceList() + const activeWorkspaces = workspaces.filter( + (workspace) => + isActiveWorkspaceStatus(workspace.status) || + workspaceJobBusy(jobs[workspace.id]), + ) + for (const [id, job] of Object.entries(jobs)) { + if ( + workspaceJobBusy(job) && + !workspaces.some((workspace) => workspace.id === id) + ) + activeWorkspaces.push({ id }) + } const template = buildTrayMenuTemplate( { activeWorkspaces, pendingStops: this.pendingStops, + jobs: this.deps.workspaceJobs?.snapshot(), updateStatus: getLastStatus(), }, { diff --git a/desktop/src/main/watcher.ts b/desktop/src/main/watcher.ts index d2779399b..d51f9361e 100644 --- a/desktop/src/main/watcher.ts +++ b/desktop/src/main/watcher.ts @@ -61,6 +61,7 @@ export class Watcher { private pollTimer: ReturnType | null = null private workspaceStatusTimer: ReturnType | null = null private fsWatcher: ReturnType | null = null + private workspaceRevision = 0 private polling = false private pollQueued = false // Serializes pollProviders so a manual refreshProviders() can never @@ -131,6 +132,12 @@ export class Watcher { this.pollMachines(), this.pollContexts(), ]) + for (const [id, job] of Object.entries( + this.deps.workspaceJobs.snapshot(), + )) { + if (job.state === "reconciling" && job.refreshError) + void this.deps.workspaceJobs.retryRefresh(id) + } } finally { this.polling = false if (this.pollQueued) { @@ -156,11 +163,11 @@ export class Watcher { /** Re-read the workspace list from disk now, without waiting for the next scheduled poll. */ async refreshWorkspaces(): Promise { - await this.queueWorkspacePoll() + await this.queueWorkspacePoll(true) } async refreshWorkspaceStatus(workspaceId: string): Promise { - await this.queueWorkspaceStatusPoll([workspaceId]) + await this.queueWorkspaceStatusPoll([workspaceId], true) } async refreshWorkspaceStatuses(): Promise { @@ -175,24 +182,32 @@ export class Watcher { void this.refreshWorkspaceStatuses() } - private queueWorkspaceStatusPoll(workspaceIds?: string[]): Promise { + private queueWorkspaceStatusPoll( + workspaceIds?: string[], + strict = false, + ): Promise { const run = this.workspaceStatusChain.then(() => - this.pollWorkspaceStatuses(workspaceIds), + this.pollWorkspaceStatuses(workspaceIds, strict), ) - this.workspaceStatusChain = run + this.workspaceStatusChain = run.catch(() => {}) return run } - private async pollWorkspaceStatuses(workspaceIds?: string[]): Promise { + private async pollWorkspaceStatuses( + workspaceIds?: string[], + strict = false, + ): Promise { this.workspaceStatusPolling = true let changed = false try { - const ids = workspaceIds ?? this.deps.state.workspaceList().map((ws) => ws.id) + const ids = + workspaceIds ?? this.deps.state.workspaceList().map((ws) => ws.id) const concurrency = 6 let next = 0 const worker = async (): Promise => { while (next < ids.length) { const id = ids[next++] + const generation = this.deps.workspaceJobs.generation(id) try { const raw = await this.deps.cli.runRaw([ "workspace", @@ -203,11 +218,19 @@ export class Watcher { "--timeout", "15s", ]) + if (generation !== this.deps.workspaceJobs.generation(id)) { + if (strict) + throw new Error("Workspace operation changed during refresh") + continue + } const status = normalizeWorkspaceStatus(raw) + if (!status && strict) + throw new Error("Workspace status unavailable") if (status && this.deps.state.updateWorkspaceStatus(id, status)) { changed = true } - } catch { + } catch (error) { + if (strict) throw error // Preserve the last known status and retry on the next sweep. } } @@ -225,13 +248,14 @@ export class Watcher { } } - private queueWorkspacePoll(): Promise { - const run = this.workspacePollChain.then(() => this.pollWorkspaces()) - this.workspacePollChain = run + private queueWorkspacePoll(strict = false): Promise { + const run = this.workspacePollChain.then(() => this.pollWorkspaces(strict)) + this.workspacePollChain = run.catch(() => {}) return run } - private async pollWorkspaces(): Promise { + private async pollWorkspaces(strict = false): Promise { + const generation = this.deps.workspaceJobs.epoch try { const workspaces = await this.queryWithFallback( this.deps.daemon @@ -239,6 +263,11 @@ export class Watcher { : undefined, () => this.deps.cli.run(["workspace", "list", "--skip-pro"]), ) + if (generation !== this.deps.workspaceJobs.epoch) { + if (strict) + throw new Error("Workspace operation changed during refresh") + return + } const changed = this.deps.state.updateWorkspaces(workspaces as any[]) if (changed) { this.broadcastWorkspaces() @@ -247,6 +276,7 @@ export class Watcher { // Polling is best-effort, but retain diagnostics so a broken CLI or // daemon cannot make the desktop appear empty without explanation. console.warn("[watcher] workspace poll failed", error) + if (strict) throw error } } @@ -256,10 +286,15 @@ export class Watcher { * workspace as idle between the delete finishing and the list catching up. */ broadcastWorkspaces(): void { - this.send("workspaces-changed", { + this.send("workspaces-changed", this.workspaceSnapshot()) + } + + workspaceSnapshot() { + return { + revision: ++this.workspaceRevision, workspaces: this.deps.state.workspaceList(), jobs: this.deps.workspaceJobs.snapshot(), - }) + } } /** Re-read provider state from disk now, without waiting for the next scheduled poll. */ diff --git a/desktop/src/main/workspace-jobs.ts b/desktop/src/main/workspace-jobs.ts index 141dfdd60..522e01e5b 100644 --- a/desktop/src/main/workspace-jobs.ts +++ b/desktop/src/main/workspace-jobs.ts @@ -1,102 +1,133 @@ -/** - * Tracks in-flight workspace delete operations in the main process, the - * same way ProviderJobs tracks provider install/init work: workspace_delete - * is fire-and-forget from the IPC handler's perspective (it returns as soon - * as the CLI command is launched, so the log-streaming UI isn't blocked), - * so nothing else records that a delete is running. Main-process ownership - * means the "Deleting" badge survives navigating away from the list or - * reloading the window while the delete is still in flight. - */ - -export interface WorkspaceJob { - activity: "deleting" - /** Set when the job ended in failure; the job is retained so the UI can show why. */ - error?: string -} +import type { OperationStatus } from "../shared/cli-error.js" +import { + type WorkspaceActivity, + type WorkspaceJob, + workspaceJobBusy, +} from "../shared/workspace-operation.js" +export type { WorkspaceJob } from "../shared/workspace-operation.js" +/** Main-owned actions are independent of the last observed runtime status. */ export class WorkspaceJobs { private jobs = new Map() - // Lets an in-flight finish() tell whether it still owns the entry. private generations = new Map() private lastGeneration = 0 private listeners = new Set<() => void>() - private refresh?: () => Promise + private refreshing = new Set() + private refresh?: (id: string, job: WorkspaceJob) => Promise + revision = 0 - /** - * Register a callback fired after every mutation, so starting or - * finishing a delete reaches the renderer without waiting for the next - * disk poll. - */ onChange(listener: () => void): () => void { this.listeners.add(listener) return () => this.listeners.delete(listener) } - private emit(): void { + this.revision++ for (const listener of this.listeners) listener() } + get epoch(): number { + return this.lastGeneration + } + generation(id: string): number { + return this.generations.get(id) ?? 0 + } + owns(id: string, commandId: string): boolean { + return this.jobs.get(id)?.commandId === commandId + } - /** - * Begin tracking a delete, clearing any previous failure. Returns a - * generation token the caller must pass to finish() so a stale exit - * callback from an earlier delete of the same workspace can't land on - * a retry that has already superseded it. - */ - start(id: string): number { - this.jobs.set(id, { activity: "deleting" }) + start( + id: string, + activity: WorkspaceActivity = "deleting", + commandId: string = crypto.randomUUID(), + ): number { + const current = this.jobs.get(id) + const canInterrupt = + (activity === "stopping" || activity === "deleting") && + (current?.activity === "starting" || current?.activity === "creating") && + current.state === "running" + if (workspaceJobBusy(current) && !current?.error && !canInterrupt) + throw new Error("A workspace operation is already in progress") const generation = ++this.lastGeneration this.generations.set(id, generation) + this.jobs.set(id, { + commandId, + activity, + state: "running", + phase: "Preparing", + }) this.emit() return generation } - - /** Stop tracking a workspace, discarding any recorded failure. */ + phase(id: string, commandId: string, phase: string): void { + const job = this.jobs.get(id) + if (!job || job.commandId !== commandId || job.state !== "running") return + this.jobs.set(id, { ...job, phase }) + this.emit() + } + progress(id: string, commandId: string, status: OperationStatus): void { + const job = this.jobs.get(id) + if (!job || job.commandId !== commandId || job.state !== "running") return + // A child finishing does not finish its parent or the command. + const phase = + status.state === "started" + ? status.step || status.phase.replaceAll("_", " ") + : job.phase + this.jobs.set(id, { ...job, status, phase }) + this.emit() + } clear(id: string): void { - this.generations.delete(id) + this.generations.set(id, ++this.lastGeneration) if (this.jobs.delete(id)) this.emit() } - - /** - * Finish the job started under `generation`. A failure is retained so the - * UI can explain it; success drops the entry once the workspace list has - * caught up, so the card doesn't flash back to its pre-delete state for - * one poll cycle. Either way, a generation mismatch means a retry has - * already superseded this job, so the call is a no-op. - */ async finish(id: string, generation: number, error?: string): Promise { - if (this.generations.get(id) !== generation) return - - if (error) { - this.jobs.set(id, { activity: "deleting", error }) - this.emit() - return - } const job = this.jobs.get(id) - if (!job) return - if (job.error) return - - await this.refresh?.() - // refresh is a CLI round-trip; a newer job may own the entry by now. - if (this.generations.get(id) !== generation) return - - this.jobs.delete(id) - this.generations.delete(id) + if (!job || this.generation(id) !== generation || job.state !== "running") + return + this.generations.set(id, ++this.lastGeneration) // invalidate pre-completion polls + this.jobs.set(id, { + ...job, + state: "reconciling", + phase: + job.activity === "deleting" ? "Refreshing list" : "Refreshing status", + error, + }) this.emit() + await this.retryRefresh(id) } - - /** - * Supplies a way to re-read the workspace list from disk, so a finished - * job isn't cleared before the list reflects the deletion. - */ - setRefresh(refresh: () => Promise): void { + async retryRefresh(id: string): Promise { + const job = this.jobs.get(id) + if ( + !job || + job.state !== "reconciling" || + this.refreshing.has(job.commandId) + ) + return + this.refreshing.add(job.commandId) + try { + await this.refresh?.(id, job) + if (this.jobs.get(id) !== job) return + this.jobs.set(id, { + ...job, + state: job.error ? "failed" : "succeeded", + phase: job.error ? "See workspace logs for details" : "", + refreshError: undefined, + }) + } catch (error) { + if (this.jobs.get(id) !== job) return + this.jobs.set(id, { + ...job, + refreshError: error instanceof Error ? error.message : String(error), + }) + } finally { + this.refreshing.delete(job.commandId) + this.emit() + } + } + setRefresh(refresh: (id: string, job: WorkspaceJob) => Promise): void { this.refresh = refresh } - get(id: string): WorkspaceJob | undefined { return this.jobs.get(id) } - - /** Snapshot for IPC, keyed by workspace id. */ snapshot(): Record { return Object.fromEntries(this.jobs) } diff --git a/desktop/src/renderer/src/lib/components/ErrorCard.svelte b/desktop/src/renderer/src/lib/components/ErrorCard.svelte index 8cb56a55e..9c21d0bd4 100644 --- a/desktop/src/renderer/src/lib/components/ErrorCard.svelte +++ b/desktop/src/renderer/src/lib/components/ErrorCard.svelte @@ -1,6 +1,6 @@ +
+ + {#if workspaceJobBusy(job)}{/if} + {label ?? status ?? "Checking"} + + {#if phase}{phase}{/if} + {#if job?.error}{job.error}{/if} + {#if job?.refreshError} + + {/if} +
diff --git a/desktop/src/renderer/src/lib/components/workspace/WorkspaceOperation.test.ts b/desktop/src/renderer/src/lib/components/workspace/WorkspaceOperation.test.ts new file mode 100644 index 000000000..7086c9e37 --- /dev/null +++ b/desktop/src/renderer/src/lib/components/workspace/WorkspaceOperation.test.ts @@ -0,0 +1,58 @@ +import { cleanup, fireEvent, render, waitFor } from "@testing-library/svelte" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { mockInvoke, resetTauriMocks } from "$lib/__mocks__/tauri.js" +import { workspaceJobs } from "$lib/stores/workspaces.js" +import { toasts } from "$lib/stores/toasts.js" +import WorkspaceOperation from "./WorkspaceOperation.svelte" +vi.mock("$lib/stores/toasts.js", () => ({ + toasts: { success: vi.fn(), error: vi.fn() }, +})) +afterEach(cleanup) +beforeEach(() => { + resetTauriMocks() + workspaceJobs.set({}) + vi.clearAllMocks() +}) +describe("WorkspaceOperation", () => { + it("keeps the action visible over a stale runtime observation", () => { + workspaceJobs.set({ + ws: { + commandId: "delete", + activity: "deleting", + state: "running", + phase: "Closing connections", + }, + }) + const ui = render(WorkspaceOperation, { id: "ws", status: "Running" }) + expect(ui.getByText("Deleting")).toBeTruthy() + expect(ui.getByText("Closing connections")).toBeTruthy() + expect(ui.queryByText("Running")).toBeNull() + }) + it("retries observation only and reports an IPC rejection", async () => { + workspaceJobs.set({ + ws: { + commandId: "delete", + activity: "deleting", + state: "reconciling", + phase: "Refreshing list", + refreshError: "offline", + }, + }) + mockInvoke.mockRejectedValue(new Error("IPC unavailable")) + const ui = render(WorkspaceOperation, { id: "ws", status: "Running" }) + expect(ui.getByText("Deleted")).toBeTruthy() + await fireEvent.click(ui.getByRole("button", { name: "Retry refresh" })) + await waitFor(() => + expect(toasts.error).toHaveBeenCalledWith( + expect.stringContaining("IPC unavailable"), + ), + ) + expect(mockInvoke).toHaveBeenCalledWith("workspace_refresh", { + workspaceId: "ws", + }) + expect( + mockInvoke.mock.calls.some((call) => call[0] === "workspace_delete"), + ).toBe(false) + expect(ui.getByText("Deleted")).toBeTruthy() + }) +}) diff --git a/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.platform.test.ts b/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.platform.test.ts index 8e7307f61..c1bb653dc 100644 --- a/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.platform.test.ts +++ b/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.platform.test.ts @@ -37,7 +37,7 @@ vi.mock("$lib/stores/providers.js", async () => { }) vi.mock("$lib/stores/workspaces.js", async () => { const { writable } = await import("svelte/store") - return { workspaces: writable<{ id: string }[]>([]) } + return { workspaces: writable<{ id: string }[]>([]), workspaceJobs: writable({}) } }) vi.mock("$lib/stores/toasts.js", () => ({ toasts: { success: vi.fn(), error: vi.fn(), info: vi.fn() }, diff --git a/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.svelte b/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.svelte index 21043f118..76afd320f 100644 --- a/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.svelte +++ b/desktop/src/renderer/src/lib/components/workspace/WorkspaceWizard.svelte @@ -1,4 +1,6 @@