diff --git a/README.md b/README.md index 9dd0e706..6ceecc8d 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,18 @@ [](https://github.com/glincker/levelrail/commits/main) [](CONTRIBUTING.md) [](https://github.com/glincker/levelrail/discussions) +[](#status) -Levelrail is a self-hosted deployment platform whose agent talks to +Levelrail is a self-hosted, open-source PaaS: an alternative to Heroku, +Vercel, and Railway for teams who'd rather run their own deployment +platform on their own Linux boxes than rent one. Its agent talks to Docker's own Engine API directly instead of SSHing into your servers and shelling out `docker` commands, with metrics and log storage built into the core instead of a separately-installed extra. Point it at one or more Linux boxes and it turns them into a private cloud: push to a git repo, get a running app with TLS, logs, metrics, and rollback. +Built for 3-50 services across 1-10 machines, not a Kubernetes +competitor.
@@ -108,7 +113,7 @@ layout can all still change without notice.
From the team behind [thesvg](https://github.com/glincker/thesvg) (6,400+ brand SVG icons) and [theauth-go](https://github.com/glincker/theauth-go) (OAuth 2.1 auth library for Go).
-## Why not Coolify or Dokploy
+## Why not Coolify or Dokploy (self-hosted Heroku/Vercel alternative)
Most self-hosted platforms in this category drive remote servers by
SSHing in and shelling out to the `docker` CLI, then parsing its text
@@ -163,6 +168,10 @@ so the code path is the same whether you're running one node or ten.
Positioning, not a ranking. All of these are worth using; the differences
below are the ones that matter for choosing between them.
+Short version: Levelrail is the only one of these six that streams
+events instead of polling, verifies backups instead of trusting them,
+and pins rollback images so garbage collection can't eat them.
+
| Project | Node control | Orchestration | Observability | Ingress | Rollback |
| --- | --- | --- | --- | --- | --- |
| **Levelrail** | Reverse-dialed gRPC agent, no CLI shelling | Custom Go reconciler over Docker Engine API, level-triggered | Node-local metrics and logs, federated query, all shipped | Embedded Caddy, in-process | Prior images pinned so garbage collection can't orphan a rollback target, cutover gated on a real readiness probe |
diff --git a/cmd/levelrail/main.go b/cmd/levelrail/main.go
index 14ade874..94dfb586 100644
--- a/cmd/levelrail/main.go
+++ b/cmd/levelrail/main.go
@@ -487,11 +487,30 @@ func run(logger *slog.Logger) error {
logger.Warn("webhook not configured", slog.String("error", err.Error()))
}
- apiHandler, apiRouter := rootHandler(logger, b, db, telemetryDB, alertingDB, secretsManager, masterKeyFilePath, webhookHandler, client, builder, deployRecorder, logBroadcaster, deployDispatcher, backupRunner, backupVerifyRunner, agentRegistry, emailSender, scheduledTaskRunner)
+ // Created here, before the router that needs to hand it to mutating
+ // handlers (api.WithReconcileNudger), rather than down at its own
+ // SetStore/SetSource call below: both setters, and Nudge itself, are
+ // safe to call on an Engine before Run starts (Run doesn't begin
+ // until further down this same function).
+ engine := reconcile.NewEngine(logger)
+
+ apiHandler, apiRouter := rootHandler(logger, b, db, telemetryDB, alertingDB, secretsManager, masterKeyFilePath, webhookHandler, client, builder, deployRecorder, logBroadcaster, deployDispatcher, backupRunner, backupVerifyRunner, agentRegistry, emailSender, scheduledTaskRunner, engine)
httpServer := &http.Server{
Addr: httpAddr(),
Handler: apiHandler,
ReadHeaderTimeout: 10 * time.Second,
+ // ReadTimeout bounds the slowest legitimate case (a plain POST
+ // body, e.g. a large secrets/env payload), not response
+ // duration, so it's safe alongside the SSE log/deploy streams
+ // and the exec terminal's own WebSocket (Hijack takes the
+ // connection out of net/http's own timeout enforcement once
+ // upgraded). IdleTimeout only bounds a keep-alive connection
+ // sitting between requests, same reasoning. WriteTimeout is
+ // deliberately not set here: it's an absolute per-request
+ // deadline net/http cannot exempt a specific route from, and
+ // would kill every one of those same long-lived streams.
+ ReadTimeout: 30 * time.Second,
+ IdleTimeout: 120 * time.Second,
}
ingressDriver := ingressdriver.New(logger)
@@ -520,7 +539,6 @@ func run(logger *slog.Logger) error {
// daemon on every tick.
meshDNSAddr := containerDNSAddr(ctx, client, meshCfg, logger)
- engine := reconcile.NewEngine(logger)
engine.SetStore(db)
engine.SetSource(dynamicSource(dynamicSourceDeps{
db: db,
@@ -1235,12 +1253,25 @@ func osPatchCheckInterval() time.Duration {
return d
}
+// loadBrand resolves brand.yaml the same way the install script's own
+// systemd unit does (WorkingDirectory=$DATA_DIR, so defaultBrandFile's
+// "./brand.yaml" lands on the copy install.sh wrote there): correct for
+// every documented install path, but a plain file-not-found error here
+// reads as an opaque startup crash to anyone who instead just built and
+// ran the binary directly from an arbitrary directory. Turning that one
+// case into an actionable message, rather than teaching brand.Load
+// itself to guess a fallback, keeps that function's own contract simple
+// (a real path in, a real Brand or a real error out).
func loadBrand() (*brand.Brand, error) {
path := os.Getenv("APP_BRAND_FILE")
if path == "" {
path = defaultBrandFile
}
- return brand.Load(path)
+ b, err := brand.Load(path)
+ if err != nil && os.IsNotExist(errors.Unwrap(err)) {
+ return nil, fmt.Errorf("%w (running via install.sh sets this up automatically; running the binary directly needs either a brand.yaml file at %q or APP_BRAND_FILE pointing at one)", err, path)
+ }
+ return b, err
}
func loadGitHubAppManifestConfig() (githubapp.ManifestConfig, error) {
@@ -1677,12 +1708,13 @@ func buildNodeSource(db *store.DB, agentRegistry *agent.Registry) build.NodeSour
// internal/api importing internal/agent.Registry directly (see
// api.NodeRuntimeResolver's own doc comment for why this stays a
// closure over resolveNodeTransport instead of a new dependency edge).
-func rootHandler(logger *slog.Logger, b *brand.Brand, db *store.DB, telemetryDB *telemetry.DB, alertingDB *alerting.DB, secretsManager *secrets.Manager, masterKeyFilePath string, webhookHandler http.Handler, client *docker.Client, builder *deploy.Pipeline, deployRecorder *deploylog.Recorder, logBroadcaster *telemetry.LogBroadcaster, deployDispatcher *alerting.DeployDispatcher, backupRunner *backup.Runner, backupVerifyRunner *backup.VerifyRunner, agentRegistry *agent.Registry, emailSender email.Sender, scheduledTaskRunner *scheduledtask.Runner) (http.Handler, *api.Router) {
+func rootHandler(logger *slog.Logger, b *brand.Brand, db *store.DB, telemetryDB *telemetry.DB, alertingDB *alerting.DB, secretsManager *secrets.Manager, masterKeyFilePath string, webhookHandler http.Handler, client *docker.Client, builder *deploy.Pipeline, deployRecorder *deploylog.Recorder, logBroadcaster *telemetry.LogBroadcaster, deployDispatcher *alerting.DeployDispatcher, backupRunner *backup.Runner, backupVerifyRunner *backup.VerifyRunner, agentRegistry *agent.Registry, emailSender email.Sender, scheduledTaskRunner *scheduledtask.Runner, engine *reconcile.Engine) (http.Handler, *api.Router) {
dataDir := os.Getenv("APP_DATA_DIR")
if dataDir == "" {
dataDir = defaultDataDir
}
opts := []api.Option{
+ api.WithReconcileNudger(engine),
api.WithTelemetryQuerier(telemetry.NewLocalFederator(telemetryDB)),
api.WithAlertRules(alertingDB),
api.WithDeployNotifyTargets(alertingDB),
diff --git a/internal/api/apps.go b/internal/api/apps.go
index 3be8657e..3f334b1d 100644
--- a/internal/api/apps.go
+++ b/internal/api/apps.go
@@ -525,6 +525,7 @@ func (rt *Router) handleCreateApp(w http.ResponseWriter, r *http.Request) {
req.SecretEnv = desired.SecretEnv
req.Secrets = nil
+ rt.nudgeReconciler()
writeJSON(w, http.StatusCreated, req)
}
@@ -651,6 +652,7 @@ func (rt *Router) reloadAndWriteApp(w http.ResponseWriter, r *http.Request, name
writeError(w, http.StatusInternalServerError, "internal error")
return
}
+ rt.nudgeReconciler()
writeJSON(w, http.StatusOK, toAppResource(*svc))
}
@@ -847,6 +849,11 @@ func (rt *Router) handleDeleteApp(w http.ResponseWriter, r *http.Request) {
rt.deleteAppIfOrphaned(r.Context(), appID)
}
+ // teardownServiceContainers above already removes this app's own
+ // containers directly, not via the reconciler; the nudge here is for
+ // application/network-cleanup, the other controller with work to do
+ // once this app's App row is gone.
+ rt.nudgeReconciler()
w.WriteHeader(http.StatusNoContent)
}
diff --git a/internal/api/apps_compose.go b/internal/api/apps_compose.go
index 3f378f40..987ee5b3 100644
--- a/internal/api/apps_compose.go
+++ b/internal/api/apps_compose.go
@@ -122,6 +122,7 @@ func (rt *Router) handleDeployCompose(w http.ResponseWriter, r *http.Request) {
notices = append(notices, composeNoticeResult{Level: string(n.Level), Message: n.Message})
}
+ rt.nudgeReconciler()
writeJSON(w, http.StatusOK, composeDeployResponse{AppID: name, Services: out, Notices: notices})
}
diff --git a/internal/api/apps_multi.go b/internal/api/apps_multi.go
index 1d55093a..e25bc3d8 100644
--- a/internal/api/apps_multi.go
+++ b/internal/api/apps_multi.go
@@ -227,6 +227,7 @@ func (rt *Router) handleDeploySpec(w http.ResponseWriter, r *http.Request) {
if !resp.AllSucceeded {
status = http.StatusMultiStatus
}
+ rt.nudgeReconciler()
writeJSON(w, status, resp)
}
diff --git a/internal/api/databases.go b/internal/api/databases.go
index 9a427181..e9671757 100644
--- a/internal/api/databases.go
+++ b/internal/api/databases.go
@@ -326,6 +326,7 @@ func (rt *Router) handleCreateDatabase(w http.ResponseWriter, r *http.Request) {
if !rt.createDesiredDatabase(w, r, req) {
return
}
+ rt.nudgeReconciler()
writeJSON(w, http.StatusCreated, req)
}
@@ -362,6 +363,7 @@ func (rt *Router) handleDeleteDatabase(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusInternalServerError, "internal error")
return
}
+ rt.nudgeReconciler()
w.WriteHeader(http.StatusNoContent)
}
@@ -375,6 +377,7 @@ func (rt *Router) reloadAndWriteDatabase(w http.ResponseWriter, r *http.Request,
writeError(w, http.StatusInternalServerError, "internal error")
return
}
+ rt.nudgeReconciler()
writeJSON(w, http.StatusOK, rt.toDatabaseResourceWithStatus(r.Context(), *d))
}
diff --git a/internal/api/deploys.go b/internal/api/deploys.go
index cb63b6ae..5e7a8153 100644
--- a/internal/api/deploys.go
+++ b/internal/api/deploys.go
@@ -76,6 +76,7 @@ func (rt *Router) handleTriggerDeploy(w http.ResponseWriter, r *http.Request) {
rt.recordPlainDeployAttempt(r.Context(), updated, req.Image)
+ rt.nudgeReconciler()
writeJSON(w, http.StatusAccepted, toAppResource(updated))
}
diff --git a/internal/api/exec.go b/internal/api/exec.go
index 92d72b81..8d9ac8df 100644
--- a/internal/api/exec.go
+++ b/internal/api/exec.go
@@ -262,7 +262,9 @@ func (rt *Router) resolveExecContainer(w http.ResponseWriter, r *http.Request, s
}
target := application.ContainerName(svc.Name, svc.Image, svc.RestartNonce)
- state, err := nodeRuntime.InspectByName(r.Context(), target)
+ inspectCtx, cancel := context.WithTimeout(r.Context(), dockerInspectTimeout)
+ state, err := nodeRuntime.InspectByName(inspectCtx, target)
+ cancel()
if err != nil {
rt.logger.Error("api: exec app: inspect container failed",
slog.String("error", err.Error()), slog.String("name", svc.Name), slog.String("container", target))
diff --git a/internal/api/exec_test.go b/internal/api/exec_test.go
index 06aff741..567ed238 100644
--- a/internal/api/exec_test.go
+++ b/internal/api/exec_test.go
@@ -41,9 +41,17 @@ type fakeExecAppRuntime struct {
listByPrefixCalls chan struct{}
inspectByNameCalls chan struct{}
+
+ // gotInspectCtx is the context InspectByName was actually called
+ // with, captured so a test can assert a caller bounded it with its
+ // own deadline (dockerInspectTimeout) rather than passing the bare
+ // request context through, which would let an unresponsive Docker
+ // daemon hang the request forever.
+ gotInspectCtx context.Context
}
-func (f *fakeExecAppRuntime) InspectByName(_ context.Context, _ string) (*docker.ContainerState, error) {
+func (f *fakeExecAppRuntime) InspectByName(ctx context.Context, _ string) (*docker.ContainerState, error) {
+ f.gotInspectCtx = ctx
if f.inspectByNameCalls != nil {
f.inspectByNameCalls <- struct{}{}
}
diff --git a/internal/api/middleware.go b/internal/api/middleware.go
new file mode 100644
index 00000000..0e448dc2
--- /dev/null
+++ b/internal/api/middleware.go
@@ -0,0 +1,123 @@
+package api
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/base64"
+ "log/slog"
+ "net/http"
+ "runtime/debug"
+)
+
+// requestIDContextKey is unexported, the standard "don't collide with
+// another package's context key" convention: an exported string key
+// would let any other package accidentally shadow or read this value.
+type requestIDContextKey struct{}
+
+// requestIDHeader is the response (and, when a caller already set one,
+// request) header a request ID travels under: the de facto convention
+// load balancers and CDNs already use, so a request arriving with one
+// set (a reverse proxy in front of this control plane) is threaded
+// through rather than replaced.
+const requestIDHeader = "X-Request-Id"
+
+// newRequestID mints an opaque, URL-safe correlation ID, the same
+// crypto/rand-plus-base64 shape randomTokenID (tokens.go) already
+// establishes for a different kind of ID.
+func newRequestID() string {
+ buf := make([]byte, 9)
+ if _, err := rand.Read(buf); err != nil {
+ // crypto/rand failing at all is a sign of a broken host, not
+ // something a request ID's own generation should ever surface
+ // as a 500: fall back to a fixed, obviously-synthetic value so
+ // tracing degrades to "less useful" rather than the request
+ // itself failing.
+ return "req_unavailable"
+ }
+ return "req_" + base64.RawURLEncoding.EncodeToString(buf)
+}
+
+// requestIDFromContext returns the current request's ID, or "" if
+// requestIDMiddleware never ran (e.g. a unit test hitting a handler
+// directly without going through Handler()).
+func requestIDFromContext(ctx context.Context) string {
+ id, _ := ctx.Value(requestIDContextKey{}).(string)
+ return id
+}
+
+// requestIDMiddleware assigns every request a correlation ID (or reuses
+// one an upstream proxy already set), echoes it back on the response,
+// and threads it through the request context so panicRecoveryMiddleware
+// and, over time, individual handlers' own log lines can tie a symptom
+// back to the exact request that caused it, without an operator having
+// to timestamp-match across log lines.
+func requestIDMiddleware(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ id := r.Header.Get(requestIDHeader)
+ if id == "" {
+ id = newRequestID()
+ }
+ w.Header().Set(requestIDHeader, id)
+ ctx := context.WithValue(r.Context(), requestIDContextKey{}, id)
+ next.ServeHTTP(w, r.WithContext(ctx))
+ })
+}
+
+// panicRecoveryMiddleware turns an unhandled panic in any handler into a
+// logged, diagnosable slog entry (request ID, method, path, the panic
+// value, and a stack trace) plus a clean JSON 500, instead of the
+// default net/http behavior: a raw stack dump to stderr and the
+// connection closed mid-response with no body at all. This is the
+// single highest-leverage hardening gap on an admin control plane,
+// where every panicking request is by definition an operator action
+// (a deploy, a delete, a config change) that just silently failed with
+// no diagnosable trace of why.
+func panicRecoveryMiddleware(logger *slog.Logger) func(http.Handler) http.Handler {
+ return func(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer func() {
+ if rec := recover(); rec != nil {
+ logger.Error("api: panic recovered",
+ slog.String("request_id", requestIDFromContext(r.Context())),
+ slog.String("method", r.Method),
+ slog.String("path", r.URL.Path),
+ slog.Any("panic", rec),
+ slog.String("stack", string(debug.Stack())),
+ )
+ writeError(w, http.StatusInternalServerError, "internal error")
+ }
+ }()
+ next.ServeHTTP(w, r)
+ })
+ }
+}
+
+// securityHeadersMiddleware sets the response headers that cost nothing
+// to get right and directly reduce the blast radius of an XSS or
+// clickjacking attempt against a dashboard with root-level actions
+// (deploys, secrets, rollbacks) behind it. Deliberately narrow: a
+// Content-Security-Policy or Strict-Transport-Security header needs
+// verifying against the actual frontend bundle and the embedded-Caddy
+// TLS story respectively before shipping, so both are left for a
+// follow-up rather than guessed at here.
+func securityHeadersMiddleware(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ h := w.Header()
+ h.Set("X-Content-Type-Options", "nosniff")
+ h.Set("X-Frame-Options", "DENY")
+ h.Set("Referrer-Policy", "strict-origin-when-cross-origin")
+ next.ServeHTTP(w, r)
+ })
+}
+
+// handleHealthz handles GET /healthz: an unauthenticated liveness check
+// for systemd, a container orchestrator, or a load balancer, the one
+// documented exception to "every route needs auth." Deliberately just
+// "is this process alive and able to write a response," not a dependency
+// check (Docker reachability, disk space): those are already covered,
+// authenticated, and richer at GET /api/v1/system/doctor, which a
+// human or levelrail-cli doctor calls, not a health-check probe that
+// runs every few seconds.
+func (rt *Router) handleHealthz(w http.ResponseWriter, _ *http.Request) {
+ writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+}
diff --git a/internal/api/middleware_test.go b/internal/api/middleware_test.go
new file mode 100644
index 00000000..1f01b391
--- /dev/null
+++ b/internal/api/middleware_test.go
@@ -0,0 +1,118 @@
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestPanicRecoveryMiddleware_RecoversAndReturns500(t *testing.T) {
+ panicking := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
+ panic("something went wrong")
+ })
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/whatever", nil)
+
+ // Must not itself panic and crash the test: that's exactly the
+ // behavior this middleware exists to prevent from reaching the
+ // caller (net/http.Server's own request-serving goroutine).
+ panicRecoveryMiddleware(discardLogger())(panicking).ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusInternalServerError {
+ t.Errorf("status = %d, want %d", rec.Code, http.StatusInternalServerError)
+ }
+ var got apiError
+ if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+ t.Fatalf("response body is not valid JSON: %v (body: %s)", err, rec.Body.String())
+ }
+ if got.Error == "" {
+ t.Error("response body has no error message")
+ }
+}
+
+func TestPanicRecoveryMiddleware_NoPanic_PassesThrough(t *testing.T) {
+ inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ writeJSON(w, http.StatusOK, map[string]string{"ok": "yes"})
+ })
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/whatever", nil)
+ panicRecoveryMiddleware(discardLogger())(inner).ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Errorf("status = %d, want %d", rec.Code, http.StatusOK)
+ }
+}
+
+func TestRequestIDMiddleware_GeneratesIDWhenAbsent(t *testing.T) {
+ var gotFromContext string
+ inner := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+ gotFromContext = requestIDFromContext(r.Context())
+ })
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/whatever", nil)
+ requestIDMiddleware(inner).ServeHTTP(rec, req)
+
+ header := rec.Header().Get(requestIDHeader)
+ if header == "" {
+ t.Fatal("response has no X-Request-Id header")
+ }
+ if gotFromContext != header {
+ t.Errorf("requestIDFromContext() = %q, want it to match the response header %q", gotFromContext, header)
+ }
+}
+
+func TestRequestIDMiddleware_ReusesIncomingID(t *testing.T) {
+ var gotFromContext string
+ inner := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+ gotFromContext = requestIDFromContext(r.Context())
+ })
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/whatever", nil)
+ req.Header.Set(requestIDHeader, "req_from_upstream_proxy")
+ requestIDMiddleware(inner).ServeHTTP(rec, req)
+
+ if got := rec.Header().Get(requestIDHeader); got != "req_from_upstream_proxy" {
+ t.Errorf("response X-Request-Id = %q, want the incoming value preserved", got)
+ }
+ if gotFromContext != "req_from_upstream_proxy" {
+ t.Errorf("requestIDFromContext() = %q, want the incoming value", gotFromContext)
+ }
+}
+
+func TestSecurityHeadersMiddleware_SetsExpectedHeaders(t *testing.T) {
+ inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/whatever", nil)
+ securityHeadersMiddleware(inner).ServeHTTP(rec, req)
+
+ tests := map[string]string{
+ "X-Content-Type-Options": "nosniff",
+ "X-Frame-Options": "DENY",
+ "Referrer-Policy": "strict-origin-when-cross-origin",
+ }
+ for header, want := range tests {
+ if got := rec.Header().Get(header); got != want {
+ t.Errorf("%s = %q, want %q", header, got, want)
+ }
+ }
+}
+
+func TestHandleHealthz_OKWithNoAuth(t *testing.T) {
+ rt, _ := newTestRouter(t)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
+ rt.Handler().ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Errorf("status = %d, want %d, body = %s", rec.Code, http.StatusOK, rec.Body.String())
+ }
+}
diff --git a/internal/api/network.go b/internal/api/network.go
index a53df43a..a195e9bf 100644
--- a/internal/api/network.go
+++ b/internal/api/network.go
@@ -1,14 +1,25 @@
package api
import (
+ "context"
"errors"
"log/slog"
"net/http"
+ "time"
"github.com/GLINCKER/levelrail/internal/reconcile/application"
"github.com/GLINCKER/levelrail/internal/store"
)
+// dockerInspectTimeout bounds one live container-state check
+// (docker.Runtime.InspectByName) issued directly from an HTTP handler:
+// network.go, exec.go, and resources_live_apply.go all read the request
+// context, which by default carries no deadline of its own, so a
+// blackholed or overloaded Docker daemon would otherwise hang the whole
+// request (and, for network.go, the entire Network tab's page load)
+// indefinitely instead of the page degrading to "status unknown."
+const dockerInspectTimeout = 3 * time.Second
+
// networkResource is GET /api/v1/apps/{name}/network's wire shape: the
// live traffic path from Caddy ingress down to the container, for the
// frontend's Network tab. HostPort is only ever meaningful while Running
@@ -57,7 +68,9 @@ func (rt *Router) handleGetAppNetwork(w http.ResponseWriter, r *http.Request) {
// internal/reconcile/ingress already use to find a service's
// currently active container from its own desired state.
target := application.ContainerName(svc.Name, svc.Image, svc.RestartNonce)
- state, err := rt2.InspectByName(r.Context(), target)
+ inspectCtx, cancel := context.WithTimeout(r.Context(), dockerInspectTimeout)
+ state, err := rt2.InspectByName(inspectCtx, target)
+ cancel()
if err != nil || state == nil {
writeJSON(w, http.StatusOK, resp)
return
diff --git a/internal/api/network_test.go b/internal/api/network_test.go
index 1dfa45ac..7acbd01a 100644
--- a/internal/api/network_test.go
+++ b/internal/api/network_test.go
@@ -146,6 +146,31 @@ func TestHandleGetAppNetwork_NodeResolverError(t *testing.T) {
}
}
+// TestHandleGetAppNetwork_InspectBoundedByTimeout proves the live
+// InspectByName call is never handed the bare request context (which
+// carries no deadline of its own): an unresponsive Docker daemon must
+// time this request out rather than hang the Network tab's page load
+// forever.
+func TestHandleGetAppNetwork_InspectBoundedByTimeout(t *testing.T) {
+ fake := &fakeExecAppRuntime{inspectState: &docker.ContainerState{ID: "c1", Running: true}}
+ rt, db := newTestRouterWithExecRuntime(t, fake)
+ cookie := loginTestSession(t, rt, db)
+ seedExecApp(t, db)
+
+ rec := httptest.NewRecorder()
+ rt.Handler().ServeHTTP(rec, authedRequest(t, cookie, http.MethodGet, "/api/v1/apps/web/network", ""))
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d, body = %s", rec.Code, http.StatusOK, rec.Body.String())
+ }
+
+ if fake.gotInspectCtx == nil {
+ t.Fatal("InspectByName was never called")
+ }
+ if _, ok := fake.gotInspectCtx.Deadline(); !ok {
+ t.Error("InspectByName's context has no deadline, want one bounded by dockerInspectTimeout")
+ }
+}
+
func TestNetworkAppRoute_RequiresAuth(t *testing.T) {
rt, _ := newTestRouter(t)
diff --git a/internal/api/project_restart.go b/internal/api/project_restart.go
index f0839928..27a532e6 100644
--- a/internal/api/project_restart.go
+++ b/internal/api/project_restart.go
@@ -63,5 +63,6 @@ func (rt *Router) handleRestartProject(w http.ResponseWriter, r *http.Request) {
}
resp.RestartedCount = len(resp.Apps)
+ rt.nudgeReconciler()
writeJSON(w, http.StatusOK, resp)
}
diff --git a/internal/api/project_stop_start.go b/internal/api/project_stop_start.go
index 71e6c011..615cf626 100644
--- a/internal/api/project_stop_start.go
+++ b/internal/api/project_stop_start.go
@@ -84,5 +84,6 @@ func (rt *Router) handleProjectLifecycle(w http.ResponseWriter, r *http.Request,
result.SucceededDatabases = append(result.SucceededDatabases, db.Name)
}
+ rt.nudgeReconciler()
writeJSON(w, http.StatusOK, result)
}
diff --git a/internal/api/promote.go b/internal/api/promote.go
index 09c85771..75d112c5 100644
--- a/internal/api/promote.go
+++ b/internal/api/promote.go
@@ -114,6 +114,7 @@ func (rt *Router) handlePromoteApp(w http.ResponseWriter, r *http.Request) {
rt.recordInstantDeployAttempt(r.Context(), updated, res.source.Image, store.DeployAttemptSourcePromote)
+ rt.nudgeReconciler()
writeJSON(w, http.StatusAccepted, toAppResource(updated))
}
diff --git a/internal/api/resources_live_apply.go b/internal/api/resources_live_apply.go
index acc909b6..6be8e88d 100644
--- a/internal/api/resources_live_apply.go
+++ b/internal/api/resources_live_apply.go
@@ -22,7 +22,9 @@ func (rt *Router) applyResourcesLive(ctx context.Context, nodeID, containerName
if err != nil {
return false
}
- state, err := runtime.InspectByName(ctx, containerName)
+ inspectCtx, cancel := context.WithTimeout(ctx, dockerInspectTimeout)
+ state, err := runtime.InspectByName(inspectCtx, containerName)
+ cancel()
if err != nil || state == nil || !state.Running {
return false
}
diff --git a/internal/api/router.go b/internal/api/router.go
index bcbad8d6..d702ffc4 100644
--- a/internal/api/router.go
+++ b/internal/api/router.go
@@ -119,6 +119,7 @@ type Router struct {
dockerPruner DockerPruner // nil is valid: POST /system/prune returns 501, same shape as builder/secrets above
registryAuthTester RegistryAuthTester // nil is valid: POST /api/v1/registry-credentials/{id}/test returns 501, same shape as dockerPinger above
execRuntime NodeRuntimeResolver // nil is valid: POST /apps/{name}/exec returns 501, same shape as dockerPruner above
+ reconcileNudger ReconcileNudger // nil is valid: a desired-state-changing handler just waits for the next resync tick instead of nudging, same "absence degrades, never errors" shape as dockerPinger above
certs CertStore // always set, part of the core Store interface: unlike dockerPinger/images this isn't an optional plug-in, every *store.DB already has it
ingressSettings IngressSettingsStore // always set, same "core Store interface, not an optional plug-in" shape as certs above: the settings row always exists (migrations/0023's own seeded row)
domains DomainStore // always set, same shape as ingressSettings above: service_domains is always queryable, empty is a valid, non-error result
diff --git a/internal/api/router_options.go b/internal/api/router_options.go
index 6fa2f752..7877d4ad 100644
--- a/internal/api/router_options.go
+++ b/internal/api/router_options.go
@@ -282,6 +282,17 @@ func WithTelemetryQuerier(q TelemetryQuerier) Option {
return func(rt *Router) { rt.telemetry = q }
}
+// WithReconcileNudger lets desired-state-changing handlers (app/database
+// create, stop, start, restart, delete) request an immediate reconcile
+// pass right after they save, instead of every such action waiting up
+// to a full resyncInterval to visibly take effect. Without one
+// configured (the default), those handlers still work exactly as
+// before: the resync ticker and Docker's own event stream are what
+// converge desired and observed state regardless.
+func WithReconcileNudger(n ReconcileNudger) Option {
+ return func(rt *Router) { rt.reconcileNudger = n }
+}
+
// WithAlertRules enables POST/GET /api/v1/apps/{name}/alerts and DELETE
// /api/v1/apps/{name}/alerts/{id}. Without one
// configured (the default), all three routes return 501, the same
diff --git a/internal/api/routes.go b/internal/api/routes.go
index e2ef8917..0369807f 100644
--- a/internal/api/routes.go
+++ b/internal/api/routes.go
@@ -11,9 +11,15 @@ import "net/http"
// under a readable size; there is no other meaning to the split.
func (rt *Router) Handler() http.Handler {
mux := http.NewServeMux()
+ mux.HandleFunc("GET /healthz", rt.handleHealthz)
rt.registerCoreRoutes(mux)
rt.registerPlatformRoutes(mux)
- return mux
+
+ var h http.Handler = mux
+ h = securityHeadersMiddleware(h)
+ h = panicRecoveryMiddleware(rt.logger)(h)
+ h = requestIDMiddleware(h)
+ return h
}
func (rt *Router) registerCoreRoutes(mux *http.ServeMux) {
diff --git a/internal/api/store_interfaces.go b/internal/api/store_interfaces.go
index 804db9f1..b6ea4947 100644
--- a/internal/api/store_interfaces.go
+++ b/internal/api/store_interfaces.go
@@ -453,6 +453,26 @@ type DockerPinger interface {
Ping(ctx context.Context) error
}
+// ReconcileNudger is *reconcile.Engine's own Nudge method, narrowed the
+// same consumer-defined way DockerPinger is above: a handler that just
+// changed desired state calls it to request an immediate reconcile pass
+// instead of leaving an operator watching a create/stop/start/restart
+// take up to a full resyncInterval (default 30s) to visibly do anything.
+// Reconcile's own idempotent, level-triggered contract is what makes an
+// extra, unscheduled pass always safe.
+type ReconcileNudger interface {
+ Nudge()
+}
+
+// nudgeReconciler calls rt.reconcileNudger.Nudge() if one is configured,
+// the one-line nil-check every mutating handler below shares instead of
+// repeating "if rt.reconcileNudger != nil" at each call site.
+func (rt *Router) nudgeReconciler() {
+ if rt.reconcileNudger != nil {
+ rt.reconcileNudger.Nudge()
+ }
+}
+
// ImageLister is the surface GET /api/v1/apps/{name}/images needs:
// discover previously-built tags under a repo, so the deploy trigger
// form (web/src/components/DeployTriggerForm.tsx) can offer a dropdown
diff --git a/internal/reconcile/engine.go b/internal/reconcile/engine.go
index 0705ae8e..a80b9e91 100644
--- a/internal/reconcile/engine.go
+++ b/internal/reconcile/engine.go
@@ -56,6 +56,15 @@ type Engine struct {
mu sync.RWMutex
lastResult map[string]Result
lastErr map[string]error
+
+ // nudge lets a caller outside the reconcile loop (internal/api, on a
+ // desired-state-changing request) request an immediate ReconcileAll
+ // instead of waiting for the next Docker event or resync tick.
+ // Buffered 1: a nudge is a level-triggered "something changed, look
+ // again" signal, not a queue of individual requests, so any number
+ // of Nudge calls between two Run passes collapses into the one
+ // ReconcileAll that was already going to happen anyway.
+ nudge chan struct{}
}
// Source dynamically supplies the current set of controllers to reconcile,
@@ -83,6 +92,21 @@ func NewEngine(logger *slog.Logger, controllers ...Controller) *Engine {
logger: logger,
lastResult: make(map[string]Result, len(controllers)),
lastErr: make(map[string]error, len(controllers)),
+ nudge: make(chan struct{}, 1),
+ }
+}
+
+// Nudge requests an immediate ReconcileAll pass on the next Run
+// iteration, instead of waiting for the next Docker event or
+// resyncInterval tick. Safe to call from any goroutine, including
+// concurrently with Run and with other Nudge calls; a full buffer means
+// a pass is already pending, so this never blocks. Reconcile's own
+// idempotent, level-triggered contract is what makes an extra,
+// unscheduled pass always safe to run.
+func (e *Engine) Nudge() {
+ select {
+ case e.nudge <- struct{}{}:
+ default:
}
}
@@ -200,6 +224,10 @@ func (e *Engine) Run(ctx context.Context, events <-chan docker.Event, resyncInte
e.logger.Debug("resync tick")
e.ReconcileAll(ctx)
+ case <-e.nudge:
+ e.logger.Debug("nudge triggered reconcile")
+ e.ReconcileAll(ctx)
+
case ev, ok := <-events:
if !ok {
e.logger.Warn("docker event stream closed, continuing on resync ticker alone")
diff --git a/internal/reconcile/engine_test.go b/internal/reconcile/engine_test.go
index d28bbcf5..f8a1a377 100644
--- a/internal/reconcile/engine_test.go
+++ b/internal/reconcile/engine_test.go
@@ -128,6 +128,60 @@ func TestEngine_Run_ReactsToEventsAndTicks(t *testing.T) {
}
}
+// TestEngine_Run_NudgeTriggersImmediateReconcile proves a caller outside
+// the Run loop (internal/api, right after a desired-state-changing
+// request) can force a reconcile pass without waiting for the next
+// resync tick: this is what keeps an interactive create/stop/start feel
+// responsive instead of taking up to a full resyncInterval to visibly
+// take effect.
+func TestEngine_Run_NudgeTriggersImmediateReconcile(t *testing.T) {
+ c := &countingController{name: "c"}
+ e := NewEngine(testLogger(), c)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+ defer cancel()
+
+ events := make(chan docker.Event)
+ done := make(chan error, 1)
+ go func() {
+ done <- e.Run(ctx, events, time.Hour) // resync so slow it would never fire on its own within the test
+ }()
+
+ time.Sleep(10 * time.Millisecond) // let the initial ReconcileAll land
+ before := c.calls.Load()
+
+ e.Nudge()
+ time.Sleep(20 * time.Millisecond) // let the nudge-triggered pass land
+
+ cancel()
+ <-done
+
+ if got := c.calls.Load(); got <= before {
+ t.Errorf("reconcile count after Nudge = %d, want more than %d (the pre-nudge count)", got, before)
+ }
+}
+
+// TestEngine_Nudge_CollapsesBurstsIntoOnePendingReconcile proves Nudge
+// never blocks and never queues more than one pending reconcile: it's a
+// level-triggered "something changed" signal, not a request counter.
+func TestEngine_Nudge_CollapsesBurstsIntoOnePendingReconcile(t *testing.T) {
+ e := NewEngine(testLogger(), &countingController{name: "c"})
+
+ done := make(chan struct{})
+ go func() {
+ for i := 0; i < 100; i++ {
+ e.Nudge()
+ }
+ close(done)
+ }()
+
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("100 Nudge calls did not return promptly; Nudge must never block")
+ }
+}
+
func TestEngine_Run_ClosedEventChannelFallsBackToTicker(t *testing.T) {
c := &countingController{name: "c"}
e := NewEngine(testLogger(), c)