feat: production-readiness batch - reconciler responsiveness, hardening, README

README.md

@@ -8,13 +8,18 @@
 [![Last commit](https://img.shields.io/github/last-commit/glincker/levelrail)](https://github.com/glincker/levelrail/commits/main)
 [![PRs welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md)
 [![Discussions](https://img.shields.io/github/discussions/glincker/levelrail)](https://github.com/glincker/levelrail/discussions)
+[![Status: pre-release](https://img.shields.io/badge/status-pre--release-orange.svg)](#status)
 
-Levelrail is a self-hosted deployment platform whose agent talks to
+Levelrail is a self-hosted, open-source PaaS: an alternative to Heroku,
+Vercel, and Railway for teams who'd rather run their own deployment
+platform on their own Linux boxes than rent one. Its agent talks to
 Docker's own Engine API directly instead of SSHing into your servers
 and shelling out `docker` commands, with metrics and log storage built
 into the core instead of a separately-installed extra. Point it at one
 or more Linux boxes and it turns them into a private cloud: push to a
 git repo, get a running app with TLS, logs, metrics, and rollback.
+Built for 3-50 services across 1-10 machines, not a Kubernetes
+competitor.
 
 <p align="center">
   <img src="docs/assets/screenshots/app-overview.png" alt="Levelrail app overview: live metrics and deploy history in one view" width="900">
@@ -108,7 +113,7 @@ layout can all still change without notice.
 
 From the team behind [thesvg](https://github.com/glincker/thesvg) (6,400+ brand SVG icons) and [theauth-go](https://github.com/glincker/theauth-go) (OAuth 2.1 auth library for Go).
 
-## Why not Coolify or Dokploy
+## Why not Coolify or Dokploy (self-hosted Heroku/Vercel alternative)
 
 Most self-hosted platforms in this category drive remote servers by
 SSHing in and shelling out to the `docker` CLI, then parsing its text
@@ -163,6 +168,10 @@ so the code path is the same whether you're running one node or ten.
 Positioning, not a ranking. All of these are worth using; the differences
 below are the ones that matter for choosing between them.
 
+Short version: Levelrail is the only one of these six that streams
+events instead of polling, verifies backups instead of trusting them,
+and pins rollback images so garbage collection can't eat them.
+
 | Project | Node control | Orchestration | Observability | Ingress | Rollback |
 | --- | --- | --- | --- | --- | --- |
 | **Levelrail** | Reverse-dialed gRPC agent, no CLI shelling | Custom Go reconciler over Docker Engine API, level-triggered | Node-local metrics and logs, federated query, all shipped | Embedded Caddy, in-process | Prior images pinned so garbage collection can't orphan a rollback target, cutover gated on a real readiness probe |

cmd/levelrail/main.go

@@ -487,11 +487,30 @@
 		logger.Warn("webhook not configured", slog.String("error", err.Error()))
 	}
 
-	apiHandler, apiRouter := rootHandler(logger, b, db, telemetryDB, alertingDB, secretsManager, masterKeyFilePath, webhookHandler, client, builder, deployRecorder, logBroadcaster, deployDispatcher, backupRunner, backupVerifyRunner, agentRegistry, emailSender, scheduledTaskRunner)
+	// Created here, before the router that needs to hand it to mutating
+	// handlers (api.WithReconcileNudger), rather than down at its own
+	// SetStore/SetSource call below: both setters, and Nudge itself, are
+	// safe to call on an Engine before Run starts (Run doesn't begin
+	// until further down this same function).
+	engine := reconcile.NewEngine(logger)
+
+	apiHandler, apiRouter := rootHandler(logger, b, db, telemetryDB, alertingDB, secretsManager, masterKeyFilePath, webhookHandler, client, builder, deployRecorder, logBroadcaster, deployDispatcher, backupRunner, backupVerifyRunner, agentRegistry, emailSender, scheduledTaskRunner, engine)
 	httpServer := &http.Server{
 		Addr:              httpAddr(),
 		Handler:           apiHandler,
 		ReadHeaderTimeout: 10 * time.Second,
+		// ReadTimeout bounds the slowest legitimate case (a plain POST
+		// body, e.g. a large secrets/env payload), not response
+		// duration, so it's safe alongside the SSE log/deploy streams
+		// and the exec terminal's own WebSocket (Hijack takes the
+		// connection out of net/http's own timeout enforcement once
+		// upgraded). IdleTimeout only bounds a keep-alive connection
+		// sitting between requests, same reasoning. WriteTimeout is
+		// deliberately not set here: it's an absolute per-request
+		// deadline net/http cannot exempt a specific route from, and
+		// would kill every one of those same long-lived streams.
+		ReadTimeout: 30 * time.Second,
+		IdleTimeout: 120 * time.Second,
 	}
 
 	ingressDriver := ingressdriver.New(logger)
@@ -520,7 +539,6 @@
 	// daemon on every tick.
 	meshDNSAddr := containerDNSAddr(ctx, client, meshCfg, logger)
 
-	engine := reconcile.NewEngine(logger)
 	engine.SetStore(db)
 	engine.SetSource(dynamicSource(dynamicSourceDeps{
 		db:               db,
@@ -1235,12 +1253,25 @@
 	return d
 }
 
+// loadBrand resolves brand.yaml the same way the install script's own
+// systemd unit does (WorkingDirectory=$DATA_DIR, so defaultBrandFile's
+// "./brand.yaml" lands on the copy install.sh wrote there): correct for
+// every documented install path, but a plain file-not-found error here
+// reads as an opaque startup crash to anyone who instead just built and
+// ran the binary directly from an arbitrary directory. Turning that one
+// case into an actionable message, rather than teaching brand.Load
+// itself to guess a fallback, keeps that function's own contract simple
+// (a real path in, a real Brand or a real error out).
 func loadBrand() (*brand.Brand, error) {
 	path := os.Getenv("APP_BRAND_FILE")
 	if path == "" {
 		path = defaultBrandFile
 	}
-	return brand.Load(path)
+	b, err := brand.Load(path)
+	if err != nil && os.IsNotExist(errors.Unwrap(err)) {
+		return nil, fmt.Errorf("%w (running via install.sh sets this up automatically; running the binary directly needs either a brand.yaml file at %q or APP_BRAND_FILE pointing at one)", err, path)
+	}
+	return b, err
 }
 
 func loadGitHubAppManifestConfig() (githubapp.ManifestConfig, error) {
@@ -1677,12 +1708,13 @@
 // internal/api importing internal/agent.Registry directly (see
 // api.NodeRuntimeResolver's own doc comment for why this stays a
 // closure over resolveNodeTransport instead of a new dependency edge).
-func rootHandler(logger *slog.Logger, b *brand.Brand, db *store.DB, telemetryDB *telemetry.DB, alertingDB *alerting.DB, secretsManager *secrets.Manager, masterKeyFilePath string, webhookHandler http.Handler, client *docker.Client, builder *deploy.Pipeline, deployRecorder *deploylog.Recorder, logBroadcaster *telemetry.LogBroadcaster, deployDispatcher *alerting.DeployDispatcher, backupRunner *backup.Runner, backupVerifyRunner *backup.VerifyRunner, agentRegistry *agent.Registry, emailSender email.Sender, scheduledTaskRunner *scheduledtask.Runner) (http.Handler, *api.Router) {
+func rootHandler(logger *slog.Logger, b *brand.Brand, db *store.DB, telemetryDB *telemetry.DB, alertingDB *alerting.DB, secretsManager *secrets.Manager, masterKeyFilePath string, webhookHandler http.Handler, client *docker.Client, builder *deploy.Pipeline, deployRecorder *deploylog.Recorder, logBroadcaster *telemetry.LogBroadcaster, deployDispatcher *alerting.DeployDispatcher, backupRunner *backup.Runner, backupVerifyRunner *backup.VerifyRunner, agentRegistry *agent.Registry, emailSender email.Sender, scheduledTaskRunner *scheduledtask.Runner, engine *reconcile.Engine) (http.Handler, *api.Router) {
 	dataDir := os.Getenv("APP_DATA_DIR")
 	if dataDir == "" {
 		dataDir = defaultDataDir
 	}
 	opts := []api.Option{
+		api.WithReconcileNudger(engine),
 		api.WithTelemetryQuerier(telemetry.NewLocalFederator(telemetryDB)),
 		api.WithAlertRules(alertingDB),
 		api.WithDeployNotifyTargets(alertingDB),

internal/api/apps.go

@@ -525,6 +525,7 @@ func (rt *Router) handleCreateApp(w http.ResponseWriter, r *http.Request) {
 	req.SecretEnv = desired.SecretEnv
 	req.Secrets = nil
 
+	rt.nudgeReconciler()
 	writeJSON(w, http.StatusCreated, req)
 }
 
@@ -651,6 +652,7 @@ func (rt *Router) reloadAndWriteApp(w http.ResponseWriter, r *http.Request, name
 		writeError(w, http.StatusInternalServerError, "internal error")
 		return
 	}
+	rt.nudgeReconciler()
 	writeJSON(w, http.StatusOK, toAppResource(*svc))
 }
 
@@ -847,6 +849,11 @@ func (rt *Router) handleDeleteApp(w http.ResponseWriter, r *http.Request) {
 		rt.deleteAppIfOrphaned(r.Context(), appID)
 	}
 
+	// teardownServiceContainers above already removes this app's own
+	// containers directly, not via the reconciler; the nudge here is for
+	// application/network-cleanup, the other controller with work to do
+	// once this app's App row is gone.
+	rt.nudgeReconciler()
 	w.WriteHeader(http.StatusNoContent)
 }
 

internal/api/apps_compose.go

@@ -122,6 +122,7 @@ func (rt *Router) handleDeployCompose(w http.ResponseWriter, r *http.Request) {
 		notices = append(notices, composeNoticeResult{Level: string(n.Level), Message: n.Message})
 	}
 
+	rt.nudgeReconciler()
 	writeJSON(w, http.StatusOK, composeDeployResponse{AppID: name, Services: out, Notices: notices})
 }
 

internal/api/apps_multi.go

@@ -227,6 +227,7 @@ func (rt *Router) handleDeploySpec(w http.ResponseWriter, r *http.Request) {
 	if !resp.AllSucceeded {
 		status = http.StatusMultiStatus
 	}
+	rt.nudgeReconciler()
 	writeJSON(w, status, resp)
 }
 

internal/api/databases.go

@@ -326,6 +326,7 @@ func (rt *Router) handleCreateDatabase(w http.ResponseWriter, r *http.Request) {
 	if !rt.createDesiredDatabase(w, r, req) {
 		return
 	}
+	rt.nudgeReconciler()
 	writeJSON(w, http.StatusCreated, req)
 }
 
@@ -362,6 +363,7 @@ func (rt *Router) handleDeleteDatabase(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusInternalServerError, "internal error")
 		return
 	}
+	rt.nudgeReconciler()
 	w.WriteHeader(http.StatusNoContent)
 }
 
@@ -375,6 +377,7 @@ func (rt *Router) reloadAndWriteDatabase(w http.ResponseWriter, r *http.Request,
 		writeError(w, http.StatusInternalServerError, "internal error")
 		return
 	}
+	rt.nudgeReconciler()
 	writeJSON(w, http.StatusOK, rt.toDatabaseResourceWithStatus(r.Context(), *d))
 }
 

internal/api/deploys.go

@@ -76,6 +76,7 @@ func (rt *Router) handleTriggerDeploy(w http.ResponseWriter, r *http.Request) {
 
 	rt.recordPlainDeployAttempt(r.Context(), updated, req.Image)
 
+	rt.nudgeReconciler()
 	writeJSON(w, http.StatusAccepted, toAppResource(updated))
 }
 

internal/api/exec.go

@@ -262,7 +262,9 @@ func (rt *Router) resolveExecContainer(w http.ResponseWriter, r *http.Request, s
 	}
 
 	target := application.ContainerName(svc.Name, svc.Image, svc.RestartNonce)
-	state, err := nodeRuntime.InspectByName(r.Context(), target)
+	inspectCtx, cancel := context.WithTimeout(r.Context(), dockerInspectTimeout)
+	state, err := nodeRuntime.InspectByName(inspectCtx, target)
+	cancel()
 	if err != nil {
 		rt.logger.Error("api: exec app: inspect container failed",
 			slog.String("error", err.Error()), slog.String("name", svc.Name), slog.String("container", target))

internal/api/exec_test.go

@@ -41,9 +41,17 @@
 	listByPrefixCalls  chan struct{}
 
 	inspectByNameCalls chan struct{}
+
+	// gotInspectCtx is the context InspectByName was actually called
+	// with, captured so a test can assert a caller bounded it with its
+	// own deadline (dockerInspectTimeout) rather than passing the bare
+	// request context through, which would let an unresponsive Docker
+	// daemon hang the request forever.
+	gotInspectCtx context.Context
 }
 
-func (f *fakeExecAppRuntime) InspectByName(_ context.Context, _ string) (*docker.ContainerState, error) {
+func (f *fakeExecAppRuntime) InspectByName(ctx context.Context, _ string) (*docker.ContainerState, error) {
+	f.gotInspectCtx = ctx
 	if f.inspectByNameCalls != nil {
 		f.inspectByNameCalls <- struct{}{}
 	}

internal/api/middleware.go

@@ -0,0 +1,123 @@
+package api
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"log/slog"
+	"net/http"
+	"runtime/debug"
+)
+
+// requestIDContextKey is unexported, the standard "don't collide with
+// another package's context key" convention: an exported string key
+// would let any other package accidentally shadow or read this value.
+type requestIDContextKey struct{}
+
+// requestIDHeader is the response (and, when a caller already set one,
+// request) header a request ID travels under: the de facto convention
+// load balancers and CDNs already use, so a request arriving with one
+// set (a reverse proxy in front of this control plane) is threaded
+// through rather than replaced.
+const requestIDHeader = "X-Request-Id"
+
+// newRequestID mints an opaque, URL-safe correlation ID, the same
+// crypto/rand-plus-base64 shape randomTokenID (tokens.go) already
+// establishes for a different kind of ID.
+func newRequestID() string {
+	buf := make([]byte, 9)
+	if _, err := rand.Read(buf); err != nil {
+		// crypto/rand failing at all is a sign of a broken host, not
+		// something a request ID's own generation should ever surface
+		// as a 500: fall back to a fixed, obviously-synthetic value so
+		// tracing degrades to "less useful" rather than the request
+		// itself failing.
+		return "req_unavailable"
+	}
+	return "req_" + base64.RawURLEncoding.EncodeToString(buf)
+}
+
+// requestIDFromContext returns the current request's ID, or "" if
+// requestIDMiddleware never ran (e.g. a unit test hitting a handler
+// directly without going through Handler()).
+func requestIDFromContext(ctx context.Context) string {
+	id, _ := ctx.Value(requestIDContextKey{}).(string)
+	return id
+}
+
+// requestIDMiddleware assigns every request a correlation ID (or reuses
+// one an upstream proxy already set), echoes it back on the response,
+// and threads it through the request context so panicRecoveryMiddleware
+// and, over time, individual handlers' own log lines can tie a symptom
+// back to the exact request that caused it, without an operator having
+// to timestamp-match across log lines.
+func requestIDMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		id := r.Header.Get(requestIDHeader)
+		if id == "" {
+			id = newRequestID()
+		}
+		w.Header().Set(requestIDHeader, id)
+		ctx := context.WithValue(r.Context(), requestIDContextKey{}, id)
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+// panicRecoveryMiddleware turns an unhandled panic in any handler into a
+// logged, diagnosable slog entry (request ID, method, path, the panic
+// value, and a stack trace) plus a clean JSON 500, instead of the
+// default net/http behavior: a raw stack dump to stderr and the
+// connection closed mid-response with no body at all. This is the
+// single highest-leverage hardening gap on an admin control plane,
+// where every panicking request is by definition an operator action
+// (a deploy, a delete, a config change) that just silently failed with
+// no diagnosable trace of why.
+func panicRecoveryMiddleware(logger *slog.Logger) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer func() {
+				if rec := recover(); rec != nil {
+					logger.Error("api: panic recovered",
+						slog.String("request_id", requestIDFromContext(r.Context())),
+						slog.String("method", r.Method),
+						slog.String("path", r.URL.Path),
+						slog.Any("panic", rec),
+						slog.String("stack", string(debug.Stack())),
+					)
+					writeError(w, http.StatusInternalServerError, "internal error")
+				}
+			}()
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+// securityHeadersMiddleware sets the response headers that cost nothing
+// to get right and directly reduce the blast radius of an XSS or
+// clickjacking attempt against a dashboard with root-level actions
+// (deploys, secrets, rollbacks) behind it. Deliberately narrow: a
+// Content-Security-Policy or Strict-Transport-Security header needs
+// verifying against the actual frontend bundle and the embedded-Caddy
+// TLS story respectively before shipping, so both are left for a
+// follow-up rather than guessed at here.
+func securityHeadersMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		h := w.Header()
+		h.Set("X-Content-Type-Options", "nosniff")
+		h.Set("X-Frame-Options", "DENY")
+		h.Set("Referrer-Policy", "strict-origin-when-cross-origin")
+		next.ServeHTTP(w, r)
+	})
+}
+
+// handleHealthz handles GET /healthz: an unauthenticated liveness check
+// for systemd, a container orchestrator, or a load balancer, the one
+// documented exception to "every route needs auth." Deliberately just
+// "is this process alive and able to write a response," not a dependency
+// check (Docker reachability, disk space): those are already covered,
+// authenticated, and richer at GET /api/v1/system/doctor, which a
+// human or levelrail-cli doctor calls, not a health-check probe that
+// runs every few seconds.
+func (rt *Router) handleHealthz(w http.ResponseWriter, _ *http.Request) {
+	writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+}