deploy: make the OTP-signup broker + route survive redeploys

cmd/otpmail/main.go

@@ -8,6 +8,8 @@
 //	OTPMAIL_MAILDIR       message dir, point at tmpfs (default /var/otpmail)
 //	OTPMAIL_SMTP_ADDR     public SMTP listen addr (default :25)
 //	OTPMAIL_CONTROL_ADDR  private control-API listen addr (default 127.0.0.1:8025)
+//	OTPMAIL_MAX_CONNS         global concurrent :25 connection cap (default 256)
+//	OTPMAIL_MAX_CONNS_PER_IP  per-remote-IP concurrent :25 connection cap (default 8)
 package main
 
 import (
@@ -20,13 +22,17 @@ import (
 
 func main() {
 	maxBytes, _ := strconv.Atoi(os.Getenv("OTPMAIL_MAX_MSG_BYTES"))
+	maxConns, _ := strconv.Atoi(os.Getenv("OTPMAIL_MAX_CONNS"))
+	maxConnsPerIP, _ := strconv.Atoi(os.Getenv("OTPMAIL_MAX_CONNS_PER_IP"))
 	srv, err := otpmail.New(otpmail.Config{
-		Domain:      os.Getenv("OTPMAIL_DOMAIN"),
-		Token:       os.Getenv("OTPMAIL_TOKEN"),
-		Maildir:     os.Getenv("OTPMAIL_MAILDIR"),
-		SMTPAddr:    os.Getenv("OTPMAIL_SMTP_ADDR"),
-		ControlAddr: os.Getenv("OTPMAIL_CONTROL_ADDR"),
-		MaxMsgBytes: maxBytes,
+		Domain:        os.Getenv("OTPMAIL_DOMAIN"),
+		Token:         os.Getenv("OTPMAIL_TOKEN"),
+		Maildir:       os.Getenv("OTPMAIL_MAILDIR"),
+		SMTPAddr:      os.Getenv("OTPMAIL_SMTP_ADDR"),
+		ControlAddr:   os.Getenv("OTPMAIL_CONTROL_ADDR"),
+		MaxMsgBytes:   maxBytes,
+		MaxConns:      maxConns,
+		MaxConnsPerIP: maxConnsPerIP,
 	})
 	if err != nil {
 		log.Fatalf("otpmail: %v", err)

cmd/otpsignup-broker/main.go

@@ -15,18 +15,29 @@
 //	OTPSIGNUP_DB                 sqlite ledger path
 //	OTPSIGNUP_ENC_KEY            64-hex (32-byte) key sealing secrets at rest
 //	OTPSIGNUP_MAX_IDS_PER_IP     per-IP distinct-caller cap (0 = unlimited)
+//	OTPSIGNUP_MINT_COOLDOWN      min gap between mints from one IP, a Go duration
+//	                             e.g. "30s" (0/unset = no cooldown)
 package main
 
 import (
 	"log"
 	"os"
 	"strconv"
+	"time"
 
 	"github.com/pilot-protocol/app-template/internal/otpsignup"
 )
 
 func main() {
 	maxIP, _ := strconv.Atoi(os.Getenv("OTPSIGNUP_MAX_IDS_PER_IP"))
+	var mintCooldown time.Duration
+	if raw := os.Getenv("OTPSIGNUP_MINT_COOLDOWN"); raw != "" {
+		d, err := time.ParseDuration(raw)
+		if err != nil {
+			log.Fatalf("otpsignup-broker: OTPSIGNUP_MINT_COOLDOWN=%q: %v", raw, err)
+		}
+		mintCooldown = d
+	}
 	b, err := otpsignup.New(otpsignup.Config{
 		Listen:             os.Getenv("OTPSIGNUP_LISTEN"),
 		MailControlURL:     os.Getenv("OTPSIGNUP_MAIL_CONTROL_URL"),
@@ -39,6 +50,7 @@ func main() {
 		DBPath:             os.Getenv("OTPSIGNUP_DB"),
 		EncKeyHex:          os.Getenv("OTPSIGNUP_ENC_KEY"),
 		MaxIdentitiesPerIP: maxIP,
+		MintCooldown:       mintCooldown,
 	})
 	if err != nil {
 		log.Fatalf("otpsignup-broker: %v", err)

deploy/setup-broker-tls.sh

@@ -17,9 +17,16 @@ ORIGIN="${BROKER_ORIGIN:-http://127.0.0.1:8099}"
 LIVE="/etc/letsencrypt/live/$HOST"
 # Extra nginx location blocks injected before `location /` — verbatim nginx
 # config for sidecar brokers that live behind the same vhost (e.g. a signup
-# broker on another port). Supplied by startup.sh from the `broker-extra-locations`
-# instance-metadata key so a regenerated broker.conf keeps those routes instead of
-# silently dropping them. Empty ⇒ just the default `location /`.
+# broker on another port). Supplied by startup.sh as the LITERAL block content
+# (not a file path) from the `broker-extra-locations` instance-metadata key, so
+# a regenerated broker.conf keeps those routes instead of silently dropping
+# them. Empty ⇒ just the default `location /`.
+#
+# SECURITY: nginx does NOT inherit proxy_set_header directives into a location
+# block that defines any proxy_set_header of its own — each injected location
+# MUST set its own `proxy_set_header X-Real-IP $remote_addr;` (mirroring the
+# default location below) or its backend will fall back to trusting a
+# client-supplied X-Forwarded-For, defeating any per-IP Sybil cap.
 EXTRA_LOCATIONS="${BROKER_EXTRA_LOCATIONS:-}"
 
 echo "→ installing nginx + certbot"
@@ -76,7 +83,9 @@ server {
 }
 NGINX
   # Inject any sidecar location blocks BEFORE `location /` (literal — the value
-  # carries nginx vars like $host/$remote_addr that bash must not expand).
+  # carries nginx vars like $host/$remote_addr that bash must not expand). This
+  # rewrites broker.conf in place (no predictable fixed-path scratch file, so
+  # there's nothing here for a symlink/race attack to target).
   if [ -n "$EXTRA_LOCATIONS" ]; then
     EXTRA_LOCATIONS="$EXTRA_LOCATIONS" python3 - <<'PY'
 import os

deploy/startup.sh

@@ -52,8 +52,9 @@ sudo -u pilot HOME=/opt/pilot bash -c '
   cd /opt/pilot
   if [ -d app-template/.git ]; then (cd app-template && git pull --ff-only); else git clone --depth 1 https://github.com/pilot-protocol/app-template; fi
   cd app-template
-  go build -o /opt/pilot/publish-server ./cmd/publish-server
-  go build -o /opt/pilot/broker         ./cmd/broker
+  go build -o /opt/pilot/publish-server   ./cmd/publish-server
+  go build -o /opt/pilot/broker           ./cmd/broker
+  go build -o /opt/pilot/otpsignup-broker ./cmd/otpsignup-broker
   # Sidecar signup broker (installed below only when its metadata env is set).
   go build -o /opt/pilot/insforge-signup-broker ./cmd/insforge-signup-broker || true
 '
@@ -129,6 +130,40 @@ RestartSec=3
 WantedBy=multi-user.target
 UNIT
 
+# ── OTP-signup broker (optional) ────────────────────────────────────────────
+# A signed broker that mints a per-user provider key with no email/code (see
+# docs/BROKER-SIGNUP.md). It runs only when the operator supplies its config in
+# broker-env metadata (an OTPSIGNUP_* block) — so a broker VM that doesn't offer
+# signup simply doesn't run it. All app/provider/host specifics live in metadata,
+# never here. Its public route is added to nginx via broker-extra-locations
+# (below), so nothing app-specific is baked into this script.
+if grep -q '^OTPSIGNUP_' /opt/pilot/broker.env 2>/dev/null; then
+  cat >/etc/systemd/system/otpsignup-broker.service <<UNIT
+[Unit]
+Description=Pilot OTP-signup broker
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+User=pilot
+EnvironmentFile=/opt/pilot/broker.env
+# Non-secret default; everything else (mail control URL + token, provider URLs,
+# domain, at-rest key, caps) comes from broker.env metadata.
+Environment=OTPSIGNUP_LISTEN=127.0.0.1:8091
+Environment=OTPSIGNUP_DB=/opt/pilot/otpsignup-data/accounts.db
+WorkingDirectory=/opt/pilot
+# 0700: the at-rest ledger (accounts.db) holds encrypted-but-sensitive per-user
+# signup state; keep the directory unreadable by anyone but the pilot user.
+ExecStartPre=/usr/bin/install -d -m 0700 -o pilot -g pilot /opt/pilot/otpsignup-data
+ExecStart=/opt/pilot/otpsignup-broker
+Restart=always
+RestartSec=3
+
+[Install]
+WantedBy=multi-user.target
+UNIT
+fi
+
 # Sidecar: the InsForge signup broker (provisions a per-user InsForge project via
 # one managed master account). Installed ONLY when the `insforge-signup-env`
 # metadata key is set (a newline-separated INSFORGE_SIGNUP_* KEY=VALUE block,
@@ -162,11 +197,13 @@ fi
 
 systemctl daemon-reload
 systemctl enable pilot-publish pilot-broker
+[ -f /etc/systemd/system/otpsignup-broker.service ] && systemctl enable otpsignup-broker
 # RESTART (not just enable --now): on a reboot/reset systemd auto-starts the
 # previously-enabled service with the OLD on-disk binary BEFORE this script
 # rebuilds it. enable --now is then a no-op and the stale binary keeps serving.
 # An explicit restart loads the freshly-built binary every deploy.
 systemctl restart pilot-publish pilot-broker
+[ -f /etc/systemd/system/otpsignup-broker.service ] && systemctl restart otpsignup-broker
 echo "pilot-publish + pilot-broker (re)started on freshly built binaries"
 
 # Expose the broker over HTTPS via nginx + a Let's Encrypt cert (idempotent;

docs/BROKER-SIGNUP.md

@@ -123,7 +123,36 @@ tiers. Cap exposure with the per-IP mint cap and the master account's own plan.
 - The mailbox holds a **live code for seconds** — treat it as a secret: tmpfs,
   `0600`, delete on read/teardown, never log the code or the body.
 - The broker seals the provider password + key **encrypted at rest** and hands
-  the key to the adapter, which owns it in `secrets.json`.
-- Receive-only, relay-denied SMTP — no open relay, no outbound mail.
+  the key to the adapter, which owns it in `secrets.json`. The at-rest key
+  (`OTPSIGNUP_ENC_KEY`) is **required** for any persistent ledger — an unset key
+  used to fall back silently to an ephemeral one, which corrupts every existing
+  account's decryptability on the next restart; the broker now fails to start
+  instead (a `:memory:`/test-only store is still allowed to auto-generate one,
+  since nothing there needs to survive a restart).
+- Receive-only, relay-denied SMTP — no open relay, no outbound mail. The `:25`
+  listener additionally caps total concurrent connections and per-remote-IP
+  concurrent connections (`OTPMAIL_MAX_CONNS[_PER_IP]`), so it can't be used to
+  exhaust the host's goroutines/file descriptors.
 - Mind the provider's own register rate limit (often per source IP): the broker's
   egress IP is the bucket, so throttle mints if volume is high.
+- The per-IP Sybil cap (`MaxIdentitiesPerIP`) keys off the caller's IP as
+  reported by `clientIP()`, which trusts **only** `X-Real-IP` (set by our own
+  nginx from `$remote_addr`, never client-controllable) or, failing that, the
+  raw socket address — it never reads a client-supplied `X-Forwarded-For`,
+  which would otherwise let anyone mint a fresh "IP" per request and bypass the
+  cap outright. Any nginx location proxying to this broker (including
+  operator-supplied `broker-extra-locations` blocks) **must** set
+  `proxy_set_header X-Real-IP $remote_addr;` itself — nginx does not inherit
+  `proxy_set_header` into a location that defines any of its own.
+- Concurrent signup requests for the same caller are serialized on a per-caller
+  lock in `Broker.Signup`, so a duplicate/racing request can't mint (and get
+  billed for) a second provider account — it gets the same cached one back.
+- **Deferred: the broker↔mail-server control plane is bearer-token auth over a
+  private interface, not mutual TLS.** Both `otpsignup.New` and `otpmail.New`
+  enforce a floor on token length/strength in the meantime (see `minTokenLen`
+  in each package) so a weak shared secret isn't the actual boundary, but a
+  compromised/leaked token (or an attacker who reaches the private interface
+  some other way) can still call the control API directly. Real mTLS between
+  the two services is a larger deployment/cert-management change and is
+  intentionally out of scope here; track it as follow-up work before running
+  this across a less-trusted network boundary than "both services on one VM."

internal/otpmail/server.go

@@ -41,12 +41,14 @@ import (
 // Config configures a Server. All fields come from the deployment environment;
 // nothing provider- or host-specific is baked in.
 type Config struct {
-	Domain      string // the mail domain we are MX for, e.g. "sub.example.net"
-	Maildir     string // dir for per-recipient message files (point at tmpfs)
-	SMTPAddr    string // public SMTP listen address, default ":25"
-	ControlAddr string // private control-API listen address, default "127.0.0.1:8025"
-	Token       string // bearer token the control API requires (required)
-	MaxMsgBytes int    // per-message cap (default 256 KiB)
+	Domain        string // the mail domain we are MX for, e.g. "sub.example.net"
+	Maildir       string // dir for per-recipient message files (point at tmpfs)
+	SMTPAddr      string // public SMTP listen address, default ":25"
+	ControlAddr   string // private control-API listen address, default "127.0.0.1:8025"
+	Token         string // bearer token the control API requires (required)
+	MaxMsgBytes   int    // per-message cap (default 256 KiB)
+	MaxConns      int    // global concurrent :25 connection cap (default 256, DoS guard)
+	MaxConnsPerIP int    // per-remote-IP concurrent :25 connection cap (default 8, DoS guard)
 }
 
 // Server is a receive-only SMTP catch-all plus a control API.
@@ -55,6 +57,15 @@ type Server struct {
 	mu     sync.Mutex
 	active map[string]time.Time // provisioned localpart@domain -> provisioned-at
 	log    *log.Logger
+
+	// Connection admission control for the public :25 listener — an unauthenticated
+	// endpoint reachable by anyone, so it needs its own DoS guards independent of
+	// the per-message/recipient checks below: a global slot semaphore plus a
+	// per-remote-IP cap, so no single source can exhaust goroutines/fds and starve
+	// everyone else.
+	connSem  chan struct{}
+	connMu   sync.Mutex
+	connByIP map[string]int
 }
 
 // New validates cfg and returns a Server.
@@ -77,13 +88,36 @@ func New(cfg Config) (*Server, error) {
 	if cfg.MaxMsgBytes <= 0 {
 		cfg.MaxMsgBytes = 256 << 10
 	}
+	if cfg.MaxConns <= 0 {
+		cfg.MaxConns = 256
+	}
+	if cfg.MaxConnsPerIP <= 0 {
+		cfg.MaxConnsPerIP = 8
+	}
+	// The control-plane channel between the signup broker and this mail server is
+	// bearer-token auth over a private interface, not mTLS (a larger deferred
+	// design change — see docs/BROKER-SIGNUP.md#security-notes). Until then,
+	// enforce a floor on token strength so a weak/guessable shared secret isn't
+	// the actual security boundary.
+	if len(cfg.Token) < minTokenLen {
+		return nil, fmt.Errorf("otpmail: control Token must be at least %d characters (use a long random bearer token)", minTokenLen)
+	}
 	cfg.Domain = strings.ToLower(cfg.Domain)
 	if err := os.MkdirAll(cfg.Maildir, 0o700); err != nil {
 		return nil, fmt.Errorf("otpmail: maildir: %w", err)
 	}
-	return &Server{cfg: cfg, active: map[string]time.Time{}, log: log.New(os.Stderr, "otpmail ", log.LstdFlags|log.LUTC)}, nil
+	return &Server{
+		cfg:      cfg,
+		active:   map[string]time.Time{},
+		log:      log.New(os.Stderr, "otpmail ", log.LstdFlags|log.LUTC),
+		connSem:  make(chan struct{}, cfg.MaxConns),
+		connByIP: map[string]int{},
+	}, nil
 }
 
+// minTokenLen is the minimum length required of the control-API bearer token.
+const minTokenLen = 20
+
 // ListenAndServe starts the SMTP + control listeners and blocks until one fails.
 func (s *Server) ListenAndServe() error {
 	errc := make(chan error, 2)
@@ -147,8 +181,67 @@ func (s *Server) serveSMTP() error {
 		if err != nil {
 			continue
 		}
-		go s.handleSMTP(c)
+		go s.acceptSMTP(c)
+	}
+}
+
+// acceptSMTP applies connection admission control (a global cap plus a
+// per-remote-IP cap) before handing the connection to handleSMTP, then
+// releases its slot(s) once handling finishes. :25 is unauthenticated and
+// internet-facing, so without this a single source (or a botnet) can open
+// unbounded connections and exhaust goroutines/file descriptors — a cheap DoS
+// against the one mailbox this VM exists to serve.
+func (s *Server) acceptSMTP(c net.Conn) {
+	ip := remoteIP(c)
+	if !s.acquireConn(ip) {
+		s.log.Printf("reject smtp connection from %s: connection cap reached", ip)
+		c.Close()
+		return
+	}
+	defer s.releaseConn(ip)
+	s.handleSMTP(c)
+}
+
+// acquireConn reserves one global slot and one per-IP slot, atomically enough
+// that a rejected per-IP attempt gives its global slot back immediately (it
+// must not leak a slot on the rejection path). Returns false if either cap is
+// currently exhausted.
+func (s *Server) acquireConn(ip string) bool {
+	select {
+	case s.connSem <- struct{}{}:
+	default:
+		return false // global concurrent-connection cap reached
+	}
+	s.connMu.Lock()
+	if s.connByIP[ip] >= s.cfg.MaxConnsPerIP {
+		s.connMu.Unlock()
+		<-s.connSem
+		return false // per-IP concurrent-connection cap reached
+	}
+	s.connByIP[ip]++
+	s.connMu.Unlock()
+	return true
+}
+
+func (s *Server) releaseConn(ip string) {
+	s.connMu.Lock()
+	if s.connByIP[ip] <= 1 {
+		delete(s.connByIP, ip)
+	} else {
+		s.connByIP[ip]--
+	}
+	s.connMu.Unlock()
+	<-s.connSem
+}
+
+// remoteIP strips the port from a connection's remote address (falling back to
+// the raw address if it isn't host:port, e.g. some test/pipe conns).
+func remoteIP(c net.Conn) string {
+	host, _, err := net.SplitHostPort(c.RemoteAddr().String())
+	if err != nil {
+		return c.RemoteAddr().String()
 	}
+	return host
 }
 
 func (s *Server) handleSMTP(c net.Conn) {