diff --git a/README.md b/README.md index 9dd0e706..6ceecc8d 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,18 @@ [![Last commit](https://img.shields.io/github/last-commit/glincker/levelrail)](https://github.com/glincker/levelrail/commits/main) [![PRs welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md) [![Discussions](https://img.shields.io/github/discussions/glincker/levelrail)](https://github.com/glincker/levelrail/discussions) +[![Status: pre-release](https://img.shields.io/badge/status-pre--release-orange.svg)](#status) -Levelrail is a self-hosted deployment platform whose agent talks to +Levelrail is a self-hosted, open-source PaaS: an alternative to Heroku, +Vercel, and Railway for teams who'd rather run their own deployment +platform on their own Linux boxes than rent one. Its agent talks to Docker's own Engine API directly instead of SSHing into your servers and shelling out `docker` commands, with metrics and log storage built into the core instead of a separately-installed extra. Point it at one or more Linux boxes and it turns them into a private cloud: push to a git repo, get a running app with TLS, logs, metrics, and rollback. +Built for 3-50 services across 1-10 machines, not a Kubernetes +competitor.

Levelrail app overview: live metrics and deploy history in one view @@ -108,7 +113,7 @@ layout can all still change without notice. From the team behind [thesvg](https://github.com/glincker/thesvg) (6,400+ brand SVG icons) and [theauth-go](https://github.com/glincker/theauth-go) (OAuth 2.1 auth library for Go). -## Why not Coolify or Dokploy +## Why not Coolify or Dokploy (self-hosted Heroku/Vercel alternative) Most self-hosted platforms in this category drive remote servers by SSHing in and shelling out to the `docker` CLI, then parsing its text @@ -163,6 +168,10 @@ so the code path is the same whether you're running one node or ten. Positioning, not a ranking. All of these are worth using; the differences below are the ones that matter for choosing between them. +Short version: Levelrail is the only one of these six that streams +events instead of polling, verifies backups instead of trusting them, +and pins rollback images so garbage collection can't eat them. + | Project | Node control | Orchestration | Observability | Ingress | Rollback | | --- | --- | --- | --- | --- | --- | | **Levelrail** | Reverse-dialed gRPC agent, no CLI shelling | Custom Go reconciler over Docker Engine API, level-triggered | Node-local metrics and logs, federated query, all shipped | Embedded Caddy, in-process | Prior images pinned so garbage collection can't orphan a rollback target, cutover gated on a real readiness probe | diff --git a/cmd/levelrail/main.go b/cmd/levelrail/main.go index 14ade874..94dfb586 100644 --- a/cmd/levelrail/main.go +++ b/cmd/levelrail/main.go @@ -487,11 +487,30 @@ func run(logger *slog.Logger) error { logger.Warn("webhook not configured", slog.String("error", err.Error())) } - apiHandler, apiRouter := rootHandler(logger, b, db, telemetryDB, alertingDB, secretsManager, masterKeyFilePath, webhookHandler, client, builder, deployRecorder, logBroadcaster, deployDispatcher, backupRunner, backupVerifyRunner, agentRegistry, emailSender, scheduledTaskRunner) + // Created here, before the router that needs to hand it to mutating + // handlers (api.WithReconcileNudger), rather than down at its own + // SetStore/SetSource call below: both setters, and Nudge itself, are + // safe to call on an Engine before Run starts (Run doesn't begin + // until further down this same function). + engine := reconcile.NewEngine(logger) + + apiHandler, apiRouter := rootHandler(logger, b, db, telemetryDB, alertingDB, secretsManager, masterKeyFilePath, webhookHandler, client, builder, deployRecorder, logBroadcaster, deployDispatcher, backupRunner, backupVerifyRunner, agentRegistry, emailSender, scheduledTaskRunner, engine) httpServer := &http.Server{ Addr: httpAddr(), Handler: apiHandler, ReadHeaderTimeout: 10 * time.Second, + // ReadTimeout bounds the slowest legitimate case (a plain POST + // body, e.g. a large secrets/env payload), not response + // duration, so it's safe alongside the SSE log/deploy streams + // and the exec terminal's own WebSocket (Hijack takes the + // connection out of net/http's own timeout enforcement once + // upgraded). IdleTimeout only bounds a keep-alive connection + // sitting between requests, same reasoning. WriteTimeout is + // deliberately not set here: it's an absolute per-request + // deadline net/http cannot exempt a specific route from, and + // would kill every one of those same long-lived streams. + ReadTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, } ingressDriver := ingressdriver.New(logger) @@ -520,7 +539,6 @@ func run(logger *slog.Logger) error { // daemon on every tick. meshDNSAddr := containerDNSAddr(ctx, client, meshCfg, logger) - engine := reconcile.NewEngine(logger) engine.SetStore(db) engine.SetSource(dynamicSource(dynamicSourceDeps{ db: db, @@ -1235,12 +1253,25 @@ func osPatchCheckInterval() time.Duration { return d } +// loadBrand resolves brand.yaml the same way the install script's own +// systemd unit does (WorkingDirectory=$DATA_DIR, so defaultBrandFile's +// "./brand.yaml" lands on the copy install.sh wrote there): correct for +// every documented install path, but a plain file-not-found error here +// reads as an opaque startup crash to anyone who instead just built and +// ran the binary directly from an arbitrary directory. Turning that one +// case into an actionable message, rather than teaching brand.Load +// itself to guess a fallback, keeps that function's own contract simple +// (a real path in, a real Brand or a real error out). func loadBrand() (*brand.Brand, error) { path := os.Getenv("APP_BRAND_FILE") if path == "" { path = defaultBrandFile } - return brand.Load(path) + b, err := brand.Load(path) + if err != nil && os.IsNotExist(errors.Unwrap(err)) { + return nil, fmt.Errorf("%w (running via install.sh sets this up automatically; running the binary directly needs either a brand.yaml file at %q or APP_BRAND_FILE pointing at one)", err, path) + } + return b, err } func loadGitHubAppManifestConfig() (githubapp.ManifestConfig, error) { @@ -1677,12 +1708,13 @@ func buildNodeSource(db *store.DB, agentRegistry *agent.Registry) build.NodeSour // internal/api importing internal/agent.Registry directly (see // api.NodeRuntimeResolver's own doc comment for why this stays a // closure over resolveNodeTransport instead of a new dependency edge). -func rootHandler(logger *slog.Logger, b *brand.Brand, db *store.DB, telemetryDB *telemetry.DB, alertingDB *alerting.DB, secretsManager *secrets.Manager, masterKeyFilePath string, webhookHandler http.Handler, client *docker.Client, builder *deploy.Pipeline, deployRecorder *deploylog.Recorder, logBroadcaster *telemetry.LogBroadcaster, deployDispatcher *alerting.DeployDispatcher, backupRunner *backup.Runner, backupVerifyRunner *backup.VerifyRunner, agentRegistry *agent.Registry, emailSender email.Sender, scheduledTaskRunner *scheduledtask.Runner) (http.Handler, *api.Router) { +func rootHandler(logger *slog.Logger, b *brand.Brand, db *store.DB, telemetryDB *telemetry.DB, alertingDB *alerting.DB, secretsManager *secrets.Manager, masterKeyFilePath string, webhookHandler http.Handler, client *docker.Client, builder *deploy.Pipeline, deployRecorder *deploylog.Recorder, logBroadcaster *telemetry.LogBroadcaster, deployDispatcher *alerting.DeployDispatcher, backupRunner *backup.Runner, backupVerifyRunner *backup.VerifyRunner, agentRegistry *agent.Registry, emailSender email.Sender, scheduledTaskRunner *scheduledtask.Runner, engine *reconcile.Engine) (http.Handler, *api.Router) { dataDir := os.Getenv("APP_DATA_DIR") if dataDir == "" { dataDir = defaultDataDir } opts := []api.Option{ + api.WithReconcileNudger(engine), api.WithTelemetryQuerier(telemetry.NewLocalFederator(telemetryDB)), api.WithAlertRules(alertingDB), api.WithDeployNotifyTargets(alertingDB), diff --git a/internal/api/apps.go b/internal/api/apps.go index 3be8657e..3f334b1d 100644 --- a/internal/api/apps.go +++ b/internal/api/apps.go @@ -525,6 +525,7 @@ func (rt *Router) handleCreateApp(w http.ResponseWriter, r *http.Request) { req.SecretEnv = desired.SecretEnv req.Secrets = nil + rt.nudgeReconciler() writeJSON(w, http.StatusCreated, req) } @@ -651,6 +652,7 @@ func (rt *Router) reloadAndWriteApp(w http.ResponseWriter, r *http.Request, name writeError(w, http.StatusInternalServerError, "internal error") return } + rt.nudgeReconciler() writeJSON(w, http.StatusOK, toAppResource(*svc)) } @@ -847,6 +849,11 @@ func (rt *Router) handleDeleteApp(w http.ResponseWriter, r *http.Request) { rt.deleteAppIfOrphaned(r.Context(), appID) } + // teardownServiceContainers above already removes this app's own + // containers directly, not via the reconciler; the nudge here is for + // application/network-cleanup, the other controller with work to do + // once this app's App row is gone. + rt.nudgeReconciler() w.WriteHeader(http.StatusNoContent) } diff --git a/internal/api/apps_compose.go b/internal/api/apps_compose.go index 3f378f40..987ee5b3 100644 --- a/internal/api/apps_compose.go +++ b/internal/api/apps_compose.go @@ -122,6 +122,7 @@ func (rt *Router) handleDeployCompose(w http.ResponseWriter, r *http.Request) { notices = append(notices, composeNoticeResult{Level: string(n.Level), Message: n.Message}) } + rt.nudgeReconciler() writeJSON(w, http.StatusOK, composeDeployResponse{AppID: name, Services: out, Notices: notices}) } diff --git a/internal/api/apps_multi.go b/internal/api/apps_multi.go index 1d55093a..e25bc3d8 100644 --- a/internal/api/apps_multi.go +++ b/internal/api/apps_multi.go @@ -227,6 +227,7 @@ func (rt *Router) handleDeploySpec(w http.ResponseWriter, r *http.Request) { if !resp.AllSucceeded { status = http.StatusMultiStatus } + rt.nudgeReconciler() writeJSON(w, status, resp) } diff --git a/internal/api/databases.go b/internal/api/databases.go index 9a427181..e9671757 100644 --- a/internal/api/databases.go +++ b/internal/api/databases.go @@ -326,6 +326,7 @@ func (rt *Router) handleCreateDatabase(w http.ResponseWriter, r *http.Request) { if !rt.createDesiredDatabase(w, r, req) { return } + rt.nudgeReconciler() writeJSON(w, http.StatusCreated, req) } @@ -362,6 +363,7 @@ func (rt *Router) handleDeleteDatabase(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "internal error") return } + rt.nudgeReconciler() w.WriteHeader(http.StatusNoContent) } @@ -375,6 +377,7 @@ func (rt *Router) reloadAndWriteDatabase(w http.ResponseWriter, r *http.Request, writeError(w, http.StatusInternalServerError, "internal error") return } + rt.nudgeReconciler() writeJSON(w, http.StatusOK, rt.toDatabaseResourceWithStatus(r.Context(), *d)) } diff --git a/internal/api/deploys.go b/internal/api/deploys.go index cb63b6ae..5e7a8153 100644 --- a/internal/api/deploys.go +++ b/internal/api/deploys.go @@ -76,6 +76,7 @@ func (rt *Router) handleTriggerDeploy(w http.ResponseWriter, r *http.Request) { rt.recordPlainDeployAttempt(r.Context(), updated, req.Image) + rt.nudgeReconciler() writeJSON(w, http.StatusAccepted, toAppResource(updated)) } diff --git a/internal/api/exec.go b/internal/api/exec.go index 92d72b81..8d9ac8df 100644 --- a/internal/api/exec.go +++ b/internal/api/exec.go @@ -262,7 +262,9 @@ func (rt *Router) resolveExecContainer(w http.ResponseWriter, r *http.Request, s } target := application.ContainerName(svc.Name, svc.Image, svc.RestartNonce) - state, err := nodeRuntime.InspectByName(r.Context(), target) + inspectCtx, cancel := context.WithTimeout(r.Context(), dockerInspectTimeout) + state, err := nodeRuntime.InspectByName(inspectCtx, target) + cancel() if err != nil { rt.logger.Error("api: exec app: inspect container failed", slog.String("error", err.Error()), slog.String("name", svc.Name), slog.String("container", target)) diff --git a/internal/api/exec_test.go b/internal/api/exec_test.go index 06aff741..567ed238 100644 --- a/internal/api/exec_test.go +++ b/internal/api/exec_test.go @@ -41,9 +41,17 @@ type fakeExecAppRuntime struct { listByPrefixCalls chan struct{} inspectByNameCalls chan struct{} + + // gotInspectCtx is the context InspectByName was actually called + // with, captured so a test can assert a caller bounded it with its + // own deadline (dockerInspectTimeout) rather than passing the bare + // request context through, which would let an unresponsive Docker + // daemon hang the request forever. + gotInspectCtx context.Context } -func (f *fakeExecAppRuntime) InspectByName(_ context.Context, _ string) (*docker.ContainerState, error) { +func (f *fakeExecAppRuntime) InspectByName(ctx context.Context, _ string) (*docker.ContainerState, error) { + f.gotInspectCtx = ctx if f.inspectByNameCalls != nil { f.inspectByNameCalls <- struct{}{} } diff --git a/internal/api/middleware.go b/internal/api/middleware.go new file mode 100644 index 00000000..0e448dc2 --- /dev/null +++ b/internal/api/middleware.go @@ -0,0 +1,123 @@ +package api + +import ( + "context" + "crypto/rand" + "encoding/base64" + "log/slog" + "net/http" + "runtime/debug" +) + +// requestIDContextKey is unexported, the standard "don't collide with +// another package's context key" convention: an exported string key +// would let any other package accidentally shadow or read this value. +type requestIDContextKey struct{} + +// requestIDHeader is the response (and, when a caller already set one, +// request) header a request ID travels under: the de facto convention +// load balancers and CDNs already use, so a request arriving with one +// set (a reverse proxy in front of this control plane) is threaded +// through rather than replaced. +const requestIDHeader = "X-Request-Id" + +// newRequestID mints an opaque, URL-safe correlation ID, the same +// crypto/rand-plus-base64 shape randomTokenID (tokens.go) already +// establishes for a different kind of ID. +func newRequestID() string { + buf := make([]byte, 9) + if _, err := rand.Read(buf); err != nil { + // crypto/rand failing at all is a sign of a broken host, not + // something a request ID's own generation should ever surface + // as a 500: fall back to a fixed, obviously-synthetic value so + // tracing degrades to "less useful" rather than the request + // itself failing. + return "req_unavailable" + } + return "req_" + base64.RawURLEncoding.EncodeToString(buf) +} + +// requestIDFromContext returns the current request's ID, or "" if +// requestIDMiddleware never ran (e.g. a unit test hitting a handler +// directly without going through Handler()). +func requestIDFromContext(ctx context.Context) string { + id, _ := ctx.Value(requestIDContextKey{}).(string) + return id +} + +// requestIDMiddleware assigns every request a correlation ID (or reuses +// one an upstream proxy already set), echoes it back on the response, +// and threads it through the request context so panicRecoveryMiddleware +// and, over time, individual handlers' own log lines can tie a symptom +// back to the exact request that caused it, without an operator having +// to timestamp-match across log lines. +func requestIDMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get(requestIDHeader) + if id == "" { + id = newRequestID() + } + w.Header().Set(requestIDHeader, id) + ctx := context.WithValue(r.Context(), requestIDContextKey{}, id) + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// panicRecoveryMiddleware turns an unhandled panic in any handler into a +// logged, diagnosable slog entry (request ID, method, path, the panic +// value, and a stack trace) plus a clean JSON 500, instead of the +// default net/http behavior: a raw stack dump to stderr and the +// connection closed mid-response with no body at all. This is the +// single highest-leverage hardening gap on an admin control plane, +// where every panicking request is by definition an operator action +// (a deploy, a delete, a config change) that just silently failed with +// no diagnosable trace of why. +func panicRecoveryMiddleware(logger *slog.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rec := recover(); rec != nil { + logger.Error("api: panic recovered", + slog.String("request_id", requestIDFromContext(r.Context())), + slog.String("method", r.Method), + slog.String("path", r.URL.Path), + slog.Any("panic", rec), + slog.String("stack", string(debug.Stack())), + ) + writeError(w, http.StatusInternalServerError, "internal error") + } + }() + next.ServeHTTP(w, r) + }) + } +} + +// securityHeadersMiddleware sets the response headers that cost nothing +// to get right and directly reduce the blast radius of an XSS or +// clickjacking attempt against a dashboard with root-level actions +// (deploys, secrets, rollbacks) behind it. Deliberately narrow: a +// Content-Security-Policy or Strict-Transport-Security header needs +// verifying against the actual frontend bundle and the embedded-Caddy +// TLS story respectively before shipping, so both are left for a +// follow-up rather than guessed at here. +func securityHeadersMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("X-Content-Type-Options", "nosniff") + h.Set("X-Frame-Options", "DENY") + h.Set("Referrer-Policy", "strict-origin-when-cross-origin") + next.ServeHTTP(w, r) + }) +} + +// handleHealthz handles GET /healthz: an unauthenticated liveness check +// for systemd, a container orchestrator, or a load balancer, the one +// documented exception to "every route needs auth." Deliberately just +// "is this process alive and able to write a response," not a dependency +// check (Docker reachability, disk space): those are already covered, +// authenticated, and richer at GET /api/v1/system/doctor, which a +// human or levelrail-cli doctor calls, not a health-check probe that +// runs every few seconds. +func (rt *Router) handleHealthz(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} diff --git a/internal/api/middleware_test.go b/internal/api/middleware_test.go new file mode 100644 index 00000000..1f01b391 --- /dev/null +++ b/internal/api/middleware_test.go @@ -0,0 +1,118 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestPanicRecoveryMiddleware_RecoversAndReturns500(t *testing.T) { + panicking := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + panic("something went wrong") + }) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/whatever", nil) + + // Must not itself panic and crash the test: that's exactly the + // behavior this middleware exists to prevent from reaching the + // caller (net/http.Server's own request-serving goroutine). + panicRecoveryMiddleware(discardLogger())(panicking).ServeHTTP(rec, req) + + if rec.Code != http.StatusInternalServerError { + t.Errorf("status = %d, want %d", rec.Code, http.StatusInternalServerError) + } + var got apiError + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("response body is not valid JSON: %v (body: %s)", err, rec.Body.String()) + } + if got.Error == "" { + t.Error("response body has no error message") + } +} + +func TestPanicRecoveryMiddleware_NoPanic_PassesThrough(t *testing.T) { + inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"ok": "yes"}) + }) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/whatever", nil) + panicRecoveryMiddleware(discardLogger())(inner).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) + } +} + +func TestRequestIDMiddleware_GeneratesIDWhenAbsent(t *testing.T) { + var gotFromContext string + inner := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + gotFromContext = requestIDFromContext(r.Context()) + }) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/whatever", nil) + requestIDMiddleware(inner).ServeHTTP(rec, req) + + header := rec.Header().Get(requestIDHeader) + if header == "" { + t.Fatal("response has no X-Request-Id header") + } + if gotFromContext != header { + t.Errorf("requestIDFromContext() = %q, want it to match the response header %q", gotFromContext, header) + } +} + +func TestRequestIDMiddleware_ReusesIncomingID(t *testing.T) { + var gotFromContext string + inner := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + gotFromContext = requestIDFromContext(r.Context()) + }) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/whatever", nil) + req.Header.Set(requestIDHeader, "req_from_upstream_proxy") + requestIDMiddleware(inner).ServeHTTP(rec, req) + + if got := rec.Header().Get(requestIDHeader); got != "req_from_upstream_proxy" { + t.Errorf("response X-Request-Id = %q, want the incoming value preserved", got) + } + if gotFromContext != "req_from_upstream_proxy" { + t.Errorf("requestIDFromContext() = %q, want the incoming value", gotFromContext) + } +} + +func TestSecurityHeadersMiddleware_SetsExpectedHeaders(t *testing.T) { + inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/whatever", nil) + securityHeadersMiddleware(inner).ServeHTTP(rec, req) + + tests := map[string]string{ + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + "Referrer-Policy": "strict-origin-when-cross-origin", + } + for header, want := range tests { + if got := rec.Header().Get(header); got != want { + t.Errorf("%s = %q, want %q", header, got, want) + } + } +} + +func TestHandleHealthz_OKWithNoAuth(t *testing.T) { + rt, _ := newTestRouter(t) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + rt.Handler().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d, body = %s", rec.Code, http.StatusOK, rec.Body.String()) + } +} diff --git a/internal/api/network.go b/internal/api/network.go index a53df43a..a195e9bf 100644 --- a/internal/api/network.go +++ b/internal/api/network.go @@ -1,14 +1,25 @@ package api import ( + "context" "errors" "log/slog" "net/http" + "time" "github.com/GLINCKER/levelrail/internal/reconcile/application" "github.com/GLINCKER/levelrail/internal/store" ) +// dockerInspectTimeout bounds one live container-state check +// (docker.Runtime.InspectByName) issued directly from an HTTP handler: +// network.go, exec.go, and resources_live_apply.go all read the request +// context, which by default carries no deadline of its own, so a +// blackholed or overloaded Docker daemon would otherwise hang the whole +// request (and, for network.go, the entire Network tab's page load) +// indefinitely instead of the page degrading to "status unknown." +const dockerInspectTimeout = 3 * time.Second + // networkResource is GET /api/v1/apps/{name}/network's wire shape: the // live traffic path from Caddy ingress down to the container, for the // frontend's Network tab. HostPort is only ever meaningful while Running @@ -57,7 +68,9 @@ func (rt *Router) handleGetAppNetwork(w http.ResponseWriter, r *http.Request) { // internal/reconcile/ingress already use to find a service's // currently active container from its own desired state. target := application.ContainerName(svc.Name, svc.Image, svc.RestartNonce) - state, err := rt2.InspectByName(r.Context(), target) + inspectCtx, cancel := context.WithTimeout(r.Context(), dockerInspectTimeout) + state, err := rt2.InspectByName(inspectCtx, target) + cancel() if err != nil || state == nil { writeJSON(w, http.StatusOK, resp) return diff --git a/internal/api/network_test.go b/internal/api/network_test.go index 1dfa45ac..7acbd01a 100644 --- a/internal/api/network_test.go +++ b/internal/api/network_test.go @@ -146,6 +146,31 @@ func TestHandleGetAppNetwork_NodeResolverError(t *testing.T) { } } +// TestHandleGetAppNetwork_InspectBoundedByTimeout proves the live +// InspectByName call is never handed the bare request context (which +// carries no deadline of its own): an unresponsive Docker daemon must +// time this request out rather than hang the Network tab's page load +// forever. +func TestHandleGetAppNetwork_InspectBoundedByTimeout(t *testing.T) { + fake := &fakeExecAppRuntime{inspectState: &docker.ContainerState{ID: "c1", Running: true}} + rt, db := newTestRouterWithExecRuntime(t, fake) + cookie := loginTestSession(t, rt, db) + seedExecApp(t, db) + + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, authedRequest(t, cookie, http.MethodGet, "/api/v1/apps/web/network", "")) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body = %s", rec.Code, http.StatusOK, rec.Body.String()) + } + + if fake.gotInspectCtx == nil { + t.Fatal("InspectByName was never called") + } + if _, ok := fake.gotInspectCtx.Deadline(); !ok { + t.Error("InspectByName's context has no deadline, want one bounded by dockerInspectTimeout") + } +} + func TestNetworkAppRoute_RequiresAuth(t *testing.T) { rt, _ := newTestRouter(t) diff --git a/internal/api/project_restart.go b/internal/api/project_restart.go index f0839928..27a532e6 100644 --- a/internal/api/project_restart.go +++ b/internal/api/project_restart.go @@ -63,5 +63,6 @@ func (rt *Router) handleRestartProject(w http.ResponseWriter, r *http.Request) { } resp.RestartedCount = len(resp.Apps) + rt.nudgeReconciler() writeJSON(w, http.StatusOK, resp) } diff --git a/internal/api/project_stop_start.go b/internal/api/project_stop_start.go index 71e6c011..615cf626 100644 --- a/internal/api/project_stop_start.go +++ b/internal/api/project_stop_start.go @@ -84,5 +84,6 @@ func (rt *Router) handleProjectLifecycle(w http.ResponseWriter, r *http.Request, result.SucceededDatabases = append(result.SucceededDatabases, db.Name) } + rt.nudgeReconciler() writeJSON(w, http.StatusOK, result) } diff --git a/internal/api/promote.go b/internal/api/promote.go index 09c85771..75d112c5 100644 --- a/internal/api/promote.go +++ b/internal/api/promote.go @@ -114,6 +114,7 @@ func (rt *Router) handlePromoteApp(w http.ResponseWriter, r *http.Request) { rt.recordInstantDeployAttempt(r.Context(), updated, res.source.Image, store.DeployAttemptSourcePromote) + rt.nudgeReconciler() writeJSON(w, http.StatusAccepted, toAppResource(updated)) } diff --git a/internal/api/resources_live_apply.go b/internal/api/resources_live_apply.go index acc909b6..6be8e88d 100644 --- a/internal/api/resources_live_apply.go +++ b/internal/api/resources_live_apply.go @@ -22,7 +22,9 @@ func (rt *Router) applyResourcesLive(ctx context.Context, nodeID, containerName if err != nil { return false } - state, err := runtime.InspectByName(ctx, containerName) + inspectCtx, cancel := context.WithTimeout(ctx, dockerInspectTimeout) + state, err := runtime.InspectByName(inspectCtx, containerName) + cancel() if err != nil || state == nil || !state.Running { return false } diff --git a/internal/api/router.go b/internal/api/router.go index bcbad8d6..d702ffc4 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -119,6 +119,7 @@ type Router struct { dockerPruner DockerPruner // nil is valid: POST /system/prune returns 501, same shape as builder/secrets above registryAuthTester RegistryAuthTester // nil is valid: POST /api/v1/registry-credentials/{id}/test returns 501, same shape as dockerPinger above execRuntime NodeRuntimeResolver // nil is valid: POST /apps/{name}/exec returns 501, same shape as dockerPruner above + reconcileNudger ReconcileNudger // nil is valid: a desired-state-changing handler just waits for the next resync tick instead of nudging, same "absence degrades, never errors" shape as dockerPinger above certs CertStore // always set, part of the core Store interface: unlike dockerPinger/images this isn't an optional plug-in, every *store.DB already has it ingressSettings IngressSettingsStore // always set, same "core Store interface, not an optional plug-in" shape as certs above: the settings row always exists (migrations/0023's own seeded row) domains DomainStore // always set, same shape as ingressSettings above: service_domains is always queryable, empty is a valid, non-error result diff --git a/internal/api/router_options.go b/internal/api/router_options.go index 6fa2f752..7877d4ad 100644 --- a/internal/api/router_options.go +++ b/internal/api/router_options.go @@ -282,6 +282,17 @@ func WithTelemetryQuerier(q TelemetryQuerier) Option { return func(rt *Router) { rt.telemetry = q } } +// WithReconcileNudger lets desired-state-changing handlers (app/database +// create, stop, start, restart, delete) request an immediate reconcile +// pass right after they save, instead of every such action waiting up +// to a full resyncInterval to visibly take effect. Without one +// configured (the default), those handlers still work exactly as +// before: the resync ticker and Docker's own event stream are what +// converge desired and observed state regardless. +func WithReconcileNudger(n ReconcileNudger) Option { + return func(rt *Router) { rt.reconcileNudger = n } +} + // WithAlertRules enables POST/GET /api/v1/apps/{name}/alerts and DELETE // /api/v1/apps/{name}/alerts/{id}. Without one // configured (the default), all three routes return 501, the same diff --git a/internal/api/routes.go b/internal/api/routes.go index e2ef8917..0369807f 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -11,9 +11,15 @@ import "net/http" // under a readable size; there is no other meaning to the split. func (rt *Router) Handler() http.Handler { mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", rt.handleHealthz) rt.registerCoreRoutes(mux) rt.registerPlatformRoutes(mux) - return mux + + var h http.Handler = mux + h = securityHeadersMiddleware(h) + h = panicRecoveryMiddleware(rt.logger)(h) + h = requestIDMiddleware(h) + return h } func (rt *Router) registerCoreRoutes(mux *http.ServeMux) { diff --git a/internal/api/store_interfaces.go b/internal/api/store_interfaces.go index 804db9f1..b6ea4947 100644 --- a/internal/api/store_interfaces.go +++ b/internal/api/store_interfaces.go @@ -453,6 +453,26 @@ type DockerPinger interface { Ping(ctx context.Context) error } +// ReconcileNudger is *reconcile.Engine's own Nudge method, narrowed the +// same consumer-defined way DockerPinger is above: a handler that just +// changed desired state calls it to request an immediate reconcile pass +// instead of leaving an operator watching a create/stop/start/restart +// take up to a full resyncInterval (default 30s) to visibly do anything. +// Reconcile's own idempotent, level-triggered contract is what makes an +// extra, unscheduled pass always safe. +type ReconcileNudger interface { + Nudge() +} + +// nudgeReconciler calls rt.reconcileNudger.Nudge() if one is configured, +// the one-line nil-check every mutating handler below shares instead of +// repeating "if rt.reconcileNudger != nil" at each call site. +func (rt *Router) nudgeReconciler() { + if rt.reconcileNudger != nil { + rt.reconcileNudger.Nudge() + } +} + // ImageLister is the surface GET /api/v1/apps/{name}/images needs: // discover previously-built tags under a repo, so the deploy trigger // form (web/src/components/DeployTriggerForm.tsx) can offer a dropdown diff --git a/internal/reconcile/engine.go b/internal/reconcile/engine.go index 0705ae8e..a80b9e91 100644 --- a/internal/reconcile/engine.go +++ b/internal/reconcile/engine.go @@ -56,6 +56,15 @@ type Engine struct { mu sync.RWMutex lastResult map[string]Result lastErr map[string]error + + // nudge lets a caller outside the reconcile loop (internal/api, on a + // desired-state-changing request) request an immediate ReconcileAll + // instead of waiting for the next Docker event or resync tick. + // Buffered 1: a nudge is a level-triggered "something changed, look + // again" signal, not a queue of individual requests, so any number + // of Nudge calls between two Run passes collapses into the one + // ReconcileAll that was already going to happen anyway. + nudge chan struct{} } // Source dynamically supplies the current set of controllers to reconcile, @@ -83,6 +92,21 @@ func NewEngine(logger *slog.Logger, controllers ...Controller) *Engine { logger: logger, lastResult: make(map[string]Result, len(controllers)), lastErr: make(map[string]error, len(controllers)), + nudge: make(chan struct{}, 1), + } +} + +// Nudge requests an immediate ReconcileAll pass on the next Run +// iteration, instead of waiting for the next Docker event or +// resyncInterval tick. Safe to call from any goroutine, including +// concurrently with Run and with other Nudge calls; a full buffer means +// a pass is already pending, so this never blocks. Reconcile's own +// idempotent, level-triggered contract is what makes an extra, +// unscheduled pass always safe to run. +func (e *Engine) Nudge() { + select { + case e.nudge <- struct{}{}: + default: } } @@ -200,6 +224,10 @@ func (e *Engine) Run(ctx context.Context, events <-chan docker.Event, resyncInte e.logger.Debug("resync tick") e.ReconcileAll(ctx) + case <-e.nudge: + e.logger.Debug("nudge triggered reconcile") + e.ReconcileAll(ctx) + case ev, ok := <-events: if !ok { e.logger.Warn("docker event stream closed, continuing on resync ticker alone") diff --git a/internal/reconcile/engine_test.go b/internal/reconcile/engine_test.go index d28bbcf5..f8a1a377 100644 --- a/internal/reconcile/engine_test.go +++ b/internal/reconcile/engine_test.go @@ -128,6 +128,60 @@ func TestEngine_Run_ReactsToEventsAndTicks(t *testing.T) { } } +// TestEngine_Run_NudgeTriggersImmediateReconcile proves a caller outside +// the Run loop (internal/api, right after a desired-state-changing +// request) can force a reconcile pass without waiting for the next +// resync tick: this is what keeps an interactive create/stop/start feel +// responsive instead of taking up to a full resyncInterval to visibly +// take effect. +func TestEngine_Run_NudgeTriggersImmediateReconcile(t *testing.T) { + c := &countingController{name: "c"} + e := NewEngine(testLogger(), c) + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + events := make(chan docker.Event) + done := make(chan error, 1) + go func() { + done <- e.Run(ctx, events, time.Hour) // resync so slow it would never fire on its own within the test + }() + + time.Sleep(10 * time.Millisecond) // let the initial ReconcileAll land + before := c.calls.Load() + + e.Nudge() + time.Sleep(20 * time.Millisecond) // let the nudge-triggered pass land + + cancel() + <-done + + if got := c.calls.Load(); got <= before { + t.Errorf("reconcile count after Nudge = %d, want more than %d (the pre-nudge count)", got, before) + } +} + +// TestEngine_Nudge_CollapsesBurstsIntoOnePendingReconcile proves Nudge +// never blocks and never queues more than one pending reconcile: it's a +// level-triggered "something changed" signal, not a request counter. +func TestEngine_Nudge_CollapsesBurstsIntoOnePendingReconcile(t *testing.T) { + e := NewEngine(testLogger(), &countingController{name: "c"}) + + done := make(chan struct{}) + go func() { + for i := 0; i < 100; i++ { + e.Nudge() + } + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("100 Nudge calls did not return promptly; Nudge must never block") + } +} + func TestEngine_Run_ClosedEventChannelFallsBackToTicker(t *testing.T) { c := &countingController{name: "c"} e := NewEngine(testLogger(), c)