feat(packages): update Docker from the dashboard

main.go

@@ -126,6 +126,14 @@ func main() {
 	alerts.Default.Latest = func() string { return service.MyService.Casa().GetCasaosVersion().Version }
 	alerts.Default.AutoUpdates = func() bool { return autoupdate.Default.Status().Enabled }
 	autoupdate.Default.Notify = alerts.Default.AutoUpdate
+	// The update of Docker stops every container, and each app's watch says so: those events are
+	// dropped while it runs and for ten minutes after. When a run ends the owner is told once, from
+	// the run's own log and systemd, so with the dashboard closed and across a restart of the core:
+	// the alerts look when the core starts (a run that ended or goes on) and when the button
+	// starts a run. Set before Run, which starts the first look.
+	alerts.Default.Muted = service.MyService.System().DockerUpdateMuted
+	alerts.Default.LastDockerRun = lastDockerRun
+	service.MyService.System().OnDockerUpdateQueued(alerts.Default.WatchDockerUpdate)
 	go alerts.Default.Run(context.Background())
 	// Automatic updates start the button's own update, read the button's own
 	// version.json, and settle an update that restarted the core before checking.
@@ -138,6 +146,19 @@ func main() {
 		return err
 	}
 	autoupdate.Default.Busy = service.MyService.System().MaintenanceBusy
+	// The update of Docker stops every container: it waits for the apps that AppManagement says
+	// are being installed, updated, backed up or restored, and does not start when it cannot ask.
+	service.MyService.System().SetAppOperations(func(ctx context.Context) ([]string, error) {
+		operations, err := autoupdate.AppOperations(ctx, config.CommonInfo.RuntimePath)
+		if err != nil {
+			return nil, err
+		}
+		apps := make([]string, 0, len(operations))
+		for _, operation := range operations {
+			apps = append(apps, operation.App)
+		}
+		return apps, nil
+	})
 	go autoupdate.Default.Run(context.Background())
 	v1Router := route.InitV1Router()
 
@@ -272,6 +293,17 @@ func main() {
 	}
 }
 
+// lastDockerRun is the last run of the Docker update as the alerts tell it, read from the run's
+// log and from systemd: it asks Docker nothing and takes no lock of the update.
+func lastDockerRun() alerts.DockerRun {
+	status, nonce := service.MyService.System().DockerUpdateRun()
+	run := alerts.DockerRun{Nonce: nonce, Outcome: status.Outcome, ErrorCode: status.ErrorCode, To: status.To}
+	for _, container := range status.NotReturned {
+		run.NotReturned = append(run.NotReturned, alerts.DockerNotReturned{Name: container.Name, RestartPolicy: container.RestartPolicy})
+	}
+	return run
+}
+
 // dashboardAddress is the dashboard's address on the box's first network, or
 // "" when it has none.
 func dashboardAddress() string {

pkg/alerts/alerts.go

@@ -11,6 +11,12 @@
 // one "resolved" message, only if its alert was sent and once per alert sent.
 // That memory is the core's: a restart forgets it, and at worst a reminder
 // comes early.
+//
+// Two things are about the update of Docker, which stops every container. While
+// it runs and just after, the apps' container events are dropped (Hub.Muted),
+// raise and resolve alike. And when it ends the owner is told once, whether or
+// not the dashboard is open, even across a restart of the core: the nonce of
+// the last run told is kept in docker-update.notified.
 package alerts
 
 import (
@@ -87,12 +93,21 @@ type Hub struct {
 	Latest      func() string
 	Current     func() string
 	AutoUpdates func() bool
+	// Muted says whether the apps' container events are to be ignored now: a Docker update stops
+	// every container, and each app's watch says so. It is asked for each of those events, so it
+	// must be cheap.
+	Muted func() bool
+	// LastDockerRun is the last run of the Docker update, from the run's log and systemd.
+	LastDockerRun func() DockerRun
 
 	modifying   sync.Mutex // serialises Update's read-modify-writes of alerts.json
 	mu          sync.Mutex // guards sent and lastFailure
 	sent        map[string]*record
 	lastFailure *Failure
 
+	dockerMu       sync.Mutex // serialises dockerEnded
+	dockerNotified string     // the nonce of the last Docker update told, guarded by dockerMu
+
 	// Read and written by Run's goroutine only.
 	seen           []disk    // the disks of the previous poll, nil before the first
 	releaseChecked time.Time // the last look for a newer release
@@ -105,26 +120,30 @@ type Hub struct {
 var Default = New("/")
 
 // New keeps alerts.json under root and sends through Shoutrrr. Address,
-// Latest and AutoUpdates are left to the caller.
+// Latest, AutoUpdates, Muted and LastDockerRun are left to the caller.
 func New(root string) *Hub {
 	return &Hub{
-		Root:        root,
-		RuntimePath: func() string { return config.CommonInfo.RuntimePath },
-		Now:         time.Now,
-		Hostname:    os.Hostname,
-		Address:     func() string { return "" },
-		Send:        send,
-		Latest:      func() string { return "" },
-		Current:     version.CurrentVersion,
-		AutoUpdates: func() bool { return true },
-		sent:        map[string]*record{},
+		Root:          root,
+		RuntimePath:   func() string { return config.CommonInfo.RuntimePath },
+		Now:           time.Now,
+		Hostname:      os.Hostname,
+		Address:       func() string { return "" },
+		Send:          send,
+		Latest:        func() string { return "" },
+		Current:       version.CurrentVersion,
+		AutoUpdates:   func() bool { return true },
+		Muted:         func() bool { return false },
+		LastDockerRun: func() DockerRun { return DockerRun{} },
+		sent:          map[string]*record{},
 	}
 }
 
-// Run follows the message bus until ctx is done, and looks at the disks and
-// for a newer release five minutes after the start, then every hour.
+// Run follows the message bus until ctx is done, tells a Docker update that ended
+// or is going on, and looks at the disks and for a newer release five minutes
+// after the start, then every hour.
 func (h *Hub) Run(ctx context.Context) {
 	go h.followBus(ctx)
+	go h.watchDocker(ctx)
 	timer := time.NewTimer(5 * time.Minute)
 	defer timer.Stop()
 	for {

pkg/alerts/bus.go

@@ -41,9 +41,13 @@ var operations = map[string]string{
 // tests can shorten it.
 var recheckRegistered = time.Minute
 
+// containerEvents are the events of AppManagement's watch of the containers. They are the ones a
+// Docker update stirs up, and so the only ones Hub.Muted silences.
+var containerEvents = []string{"app:container-died", "app:container-unhealthy", "app:container-restarting", "app:container-healthy"}
+
 // busEvents are the events of AppManagement this package subscribes to.
 func busEvents() []string {
-	names := []string{"backup:error", "app:container-died", "app:container-unhealthy", "app:container-restarting", "app:container-healthy"}
+	names := append([]string{"backup:error"}, containerEvents...)
 	for operation := range operations {
 		names = append(names, "app:"+operation+"-error")
 	}
@@ -158,8 +162,14 @@ func registered(ctx context.Context, address, runtimePath string) ([]string, err
 	return names, nil
 }
 
-// onEvent raises or resolves the alert of an event.
+// onEvent raises or resolves the alert of an event. While Docker is being updated, and just
+// after, a container event is dropped, raise or resolve: it is Docker restarting. Dropped is
+// dropped, the Hub's memory of what it sent is not touched, so that no "resolved" follows an
+// alert that was never sent.
 func (h *Hub) onEvent(name string, properties map[string]string) {
+	if slices.Contains(containerEvents, name) && h.Muted() {
+		return
+	}
 	a, resolved, ok := fromEvent(name, properties)
 	switch {
 	case !ok:

pkg/alerts/docker.go

@@ -0,0 +1,174 @@
+package alerts
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ReCasaOS/CasaOS-Common/utils/logger"
+	"github.com/ReCasaOS/CasaOS/pkg/dockerpkg"
+	"github.com/ReCasaOS/CasaOS/pkg/utils/file"
+	"go.uber.org/zap"
+)
+
+// DockerNotifiedFile holds the nonce of the last Docker update that was told to the owner. It is
+// the core's only memory of it: a restart of the core, at any moment of a run, changes nothing.
+const DockerNotifiedFile = "/var/lib/casaos/docker-update.notified"
+
+var (
+	// dockerPoll is how often the watcher looks for the end of a run, and dockerPolls how many
+	// times it looks: 5 seconds for 2 hours, more than any update takes. A run that outlasts it
+	// is told by the next start of the core. Vars, so tests can shorten them.
+	dockerPoll  = 5 * time.Second
+	dockerPolls = 1440
+)
+
+// maxNotReturned is how many container names a message lists before it says "and N more".
+const maxNotReturned = 10
+
+// DockerRun is a run of the Docker update as far as the owner's phone is concerned. The service
+// reads it from the run's log and from systemd; nothing here talks to Docker.
+type DockerRun struct {
+	// Nonce names the run, and is "" when there is no run to tell of.
+	Nonce string
+	// Outcome is "success", "restart_pending" or "failed" once the run has ended, and "" while it
+	// has not.
+	Outcome string
+	// ErrorCode is why a run failed: guard, download, install, daemon, start or no_result.
+	ErrorCode string
+	// To is the version of the engine that runs after a run that succeeded.
+	To string
+	// NotReturned are the containers that were running before the update and were not after,
+	// already validated by whoever read the log.
+	NotReturned []DockerNotReturned
+}
+
+// DockerNotReturned is a container that was running before the update and was not running when the
+// unit stopped waiting for it.
+type DockerNotReturned struct {
+	Name string
+	// RestartPolicy is Docker's. A container whose policy is "" or "no" does not start again by
+	// itself; one that has another may, and was not running when the unit stopped waiting.
+	RestartPolicy string
+}
+
+// WatchDockerUpdate is the update button's hook: a run was just started, tell the owner when it
+// ends. It returns at once.
+func (h *Hub) WatchDockerUpdate() {
+	go h.watchDocker(context.Background())
+}
+
+// watchDocker waits for the last run of the Docker update to end and tells it, looking every few
+// seconds for about two hours. It is started with the core, for a run that ended or is going on
+// while the core was down or restarting, and by the button, for the one it starts: with no run
+// to wait for it returns at once. Any number of them may run: dockerEnded tells a run once.
+func (h *Hub) watchDocker(ctx context.Context) {
+	for i := 0; i < dockerPolls; i++ {
+		run := h.LastDockerRun()
+		switch {
+		case run.Nonce == "":
+			return
+		case run.Outcome != "":
+			h.dockerEnded(run)
+			return
+		}
+		select {
+		case <-time.After(dockerPoll):
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// dockerEnded sends the one message of a run that ended, and does nothing for a run it already
+// told: the nonce of the last one is in memory and in the marker file, and the file is written
+// before the message is sent, so that no restart, however badly timed, tells a run twice. It is
+// written whether or not there is a channel to tell: the owner who adds one next week is not sent
+// the news of last week's run.
+func (h *Hub) dockerEnded(run DockerRun) {
+	h.dockerMu.Lock()
+	defer h.dockerMu.Unlock()
+	if run.Nonce == "" || run.Nonce == h.dockerNotified {
+		return
+	}
+	path := filepath.Join(h.Root, DockerNotifiedFile)
+	if data, err := os.ReadFile(path); err == nil && strings.TrimSpace(string(data)) == run.Nonce {
+		h.dockerNotified = run.Nonce
+		return
+	}
+	// ponytail: without the file (an unwritable state directory) a restart tells the run again.
+	h.dockerNotified = run.Nonce
+	if err := file.WriteFileAtomic(path, []byte(run.Nonce+"\n")); err != nil {
+		logger.Error("alerts: cannot remember that a Docker update was told", zap.Error(err))
+	}
+	// one key per run: the six hours an alert stays quiet are for repeats, not for the next run
+	h.raise(alert{key: "docker-update:" + run.Nonce, category: Updates, sentence: dockerSentence(run)})
+}
+
+// dockerSentence is what the owner reads of a run that ended: what became of Docker, then the
+// containers that are not running again.
+func dockerSentence(run DockerRun) string {
+	var sentence string
+	switch run.Outcome {
+	case "success":
+		sentence = "Docker was updated"
+		if run.To != "" {
+			sentence += " to " + run.To
+		}
+		sentence += "."
+	case "restart_pending":
+		sentence = "Docker's new version is installed, but the old one still runs: restart Docker to switch (sudo systemctl restart docker, which stops every container)."
+	default:
+		sentence = "The Docker update failed: " + dockerFailure(run.ErrorCode) + "."
+	}
+	// A container with no restart policy is said never to come back. One with a policy is only said
+	// to be down after the wait, and not promised to come back: a policy such as on-failure does
+	// not bring a container up after the daemon stopped it cleanly.
+	var stay, late []string
+	for _, container := range run.NotReturned {
+		if container.RestartPolicy == "" || container.RestartPolicy == "no" {
+			stay = append(stay, container.Name)
+		} else {
+			late = append(late, container.Name)
+		}
+	}
+	if len(stay) > 0 {
+		sentence += " Not running again: " + nameList(stay) + " (they do not start by themselves)."
+	}
+	if len(late) > 0 {
+		sentence += " Still not running after " + strconv.Itoa(dockerpkg.ReturnWaitSeconds) + " seconds: " + nameList(late) + " (they have a restart policy; if they stay stopped, start them from the dashboard)."
+	}
+	if run.Outcome == "failed" {
+		sentence += " Details are in the dashboard."
+	}
+	return sentence
+}
+
+// nameList is the names, the first maxNotReturned of them, and how many more there are.
+func nameList(names []string) string {
+	list := strings.Join(names[:min(len(names), maxNotReturned)], ", ")
+	if len(names) > maxNotReturned {
+		list += " and " + strconv.Itoa(len(names)-maxNotReturned) + " more"
+	}
+	return list
+}
+
+// dockerFailure is why a run failed, in words: the code is the unit's and nothing else is said.
+func dockerFailure(code string) string {
+	switch code {
+	case "guard":
+		return "it was no longer what you confirmed, so nothing was installed"
+	case "download":
+		return "the packages could not be downloaded, nothing was changed"
+	case "install":
+		return "the packages could not be installed"
+	case "daemon":
+		return "Docker did not come back after the update"
+	case "start":
+		return "the update could not be started, nothing was changed"
+	}
+	return "it stopped before it reported a result"
+}