feat(forge): P38 F5b structured logs, graceful drain, bind-gate tests, container image, ops guide

.dockerignore

@@ -0,0 +1,6 @@
+.git
+.worktrees
+target
+**/target
+**/node_modules
+apps/desktop/dist

.github/workflows/ci.yml

@@ -66,6 +66,32 @@ jobs:
       - name: check
         run: bash crates/forge/conformance/forge-check.sh
 
+  # Advisory (not a required context): the sc-forge container image builds and works (git new
+  # enough, boots, serves, clones, and exits 0 on SIGTERM). Only runs when the forge, its image
+  # or the lockfile changed.
+  forge-image:
+    name: sc-forge image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+      - name: changed?
+        id: changed
+        run: |
+          base="${{ github.event.pull_request.base.sha || github.event.before }}"
+          if [ -z "$base" ] || ! git diff --quiet "$base" HEAD -- crates/forge crates/core crates/crypto Cargo.toml Cargo.lock .dockerignore rust-toolchain.toml 2>/dev/null; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+      - name: build
+        if: steps.changed.outputs.run == 'true'
+        run: docker build -f crates/forge/Dockerfile -t sc-forge:ci .
+      - name: smoke
+        if: steps.changed.outputs.run == 'true'
+        run: bash crates/forge/docker/smoke.sh sc-forge:ci
+
   # Advisory (not a required context): proves the Windows build and the owner-only
   # ACL on .sc/local-key, the `sc keygen` identity and the TLS serve key (#214,
   # #281). Deliberately not `cargo test --workspace`; the wider suite still has

ROADMAP.md

@@ -707,7 +707,7 @@ and reconciler, collaborators, branches) and F2b (smart-HTTP git transport via `
 `pre-receive` ref policy, admission control, threat-model pass #1) and F3a (content reads: refs, tags, commits,
 trees, contents, archives) and F3b (contents writes: create, update, delete as server-made commits) and F4a (branch protection
 enforced on push, contents API and branch delete; commit statuses) and F4b (pull requests, reviews, comments, retention
-refs) and F4c (merge with a crash-safe journal, required-check gating, kill -9 and protection-matrix tests) and F5a (two-stage gc, merge-commit retention, runtime cleanup of stale locks and quarantine directories) are built. Hosted repos are
+refs) and F4c (merge with a crash-safe journal, required-check gating, kill -9 and protection-matrix tests) and F5a (two-stage gc, merge-commit retention, runtime cleanup of stale locks and quarantine directories) and F5b (structured logs, graceful SIGTERM drain, bind-gate tests, container image, operations guide) are built. Hosted repos are
 bare Git and get none of the `.sc/` features.
 
 Other follow-up work is tracked as GitHub issues and listed under Deferred

crates/forge/Dockerfile

@@ -0,0 +1,27 @@
+# sc-forge image. Build from the repository root:
+#   docker build -f crates/forge/Dockerfile -t sc-forge .
+# Base images are pinned by digest; bump them together with rust-toolchain.toml.
+
+FROM rust:1.96.1-bookworm@sha256:a339861ae23e9abb272cea45dfafde21760d2ce6577a70f8a926153677902663 AS build
+WORKDIR /src
+COPY . .
+RUN cargo build --release --locked -p scl-forge --bin sc-forge
+
+# trixie, not bookworm: sc-forge needs git 2.40 or newer (merge-tree --write-tree, quarantine
+# naming) and bookworm ships 2.39. No sshd, no shell tooling beyond the base.
+FROM debian:trixie-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends git ca-certificates tini \
+ && rm -rf /var/lib/apt/lists/* \
+ && useradd --system --uid 10001 --home-dir /data --shell /usr/sbin/nologin forge \
+ && mkdir /data && chown forge:forge /data
+COPY --from=build /src/target/release/sc-forge /usr/local/bin/sc-forge
+USER forge
+VOLUME /data
+EXPOSE 3000
+# SIGTERM drains (see docs/FORGE-OPERATIONS.md); give the orchestrator longer than --shutdown-timeout.
+STOPSIGNAL SIGTERM
+# tini is PID 1: it reaps the git processes that outlive a killed transport (orphans are re-parented
+# to PID 1 and would otherwise stay zombies until pids.max runs out) and forwards SIGTERM.
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/sc-forge"]
+CMD ["--help"]

crates/forge/docker/smoke.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Smoke test of the sc-forge image: it starts, answers /healthz, git is new enough, a repository
+# can be created and cloned, and `docker stop` ends it with exit code 0 (the SIGTERM drain).
+# Usage: smoke.sh <image>
+set -euo pipefail
+image=${1:?usage: smoke.sh <image>}
+name=sc-forge-smoke-$$
+port=${SMOKE_PORT:-13000}
+cleanup() { docker rm -f "$name" >/dev/null 2>&1 || true; docker volume rm -f "$name-data" >/dev/null 2>&1 || true; }
+trap cleanup EXIT
+
+docker run --rm --entrypoint git "$image" --version
+docker volume create "$name-data" >/dev/null
+docker run --rm -v "$name-data:/data" "$image" admin user create --data-dir /data \
+  --username siteadmin --password "$(head -c 24 /dev/urandom | base64)" \
+  --email admin@example.invalid --admin --must-change-password=false >/dev/null
+token=$(docker run --rm -v "$name-data:/data" "$image" admin user generate-access-token --data-dir /data \
+  --username siteadmin --token-name smoke --scopes all --raw)
+docker run -d --name "$name" -v "$name-data:/data" -p "127.0.0.1:$port:3000" "$image" serve \
+  --data-dir /data --listen 0.0.0.0:3000 --behind-proxy --public-url "http://127.0.0.1:$port" >/dev/null
+
+for _ in $(seq 1 60); do
+  curl -fsS "http://127.0.0.1:$port/api/healthz" >/dev/null 2>&1 && break
+  sleep 0.5
+done
+curl -fsS "http://127.0.0.1:$port/api/healthz" >/dev/null
+curl -fsS -H "Authorization: token $token" -H 'Content-Type: application/json' \
+  -d '{"name":"smoke","auto_init":true}' "http://127.0.0.1:$port/api/v1/user/repos" >/dev/null
+work=$(mktemp -d)
+git clone -q "http://siteadmin:$token@127.0.0.1:$port/siteadmin/smoke.git" "$work/smoke"
+test -f "$work/smoke/README.md"
+rm -rf "$work"
+
+docker stop -t 40 "$name" >/dev/null
+code=$(docker inspect -f '{{.State.ExitCode}}' "$name")
+[ "$code" = 0 ] || { echo "sc-forge exited with $code after SIGTERM" >&2; docker logs "$name" >&2; exit 1; }
+echo "smoke ok"

crates/forge/src/cgi.rs

@@ -345,7 +345,7 @@ impl Pump {
                     GROW_STEP,
                 );
                 if !ok {
-                    eprintln!("sc-forge: push aborted: the disk reserve cannot cover it");
+                    crate::log::warning!("push aborted: the disk reserve cannot cover it");
                     self.stop(ABORT_NO_SPACE);
                     return false;
                 }
@@ -453,6 +453,9 @@ fn environment(l: &Launch, marker: &std::path::Path) -> Vec<(String, String)> {
         ("uploadpackfilter.allow", "false".into()),
         ("uploadpackfilter.blob:none.allow", "true".into()),
         ("receive.maxInputSize", cfg.max_pack.to_string()),
+        // git sizes its pack threads by the host's CPUs, which a container does not narrow: bound
+        // them, or one transport on a big node starts dozens of threads (see FORGE-OPERATIONS.md).
+        ("pack.threads", "4".into()),
         ("receive.autogc", "false".into()),
         ("receive.advertisePushOptions", "true".into()),
     ];
@@ -815,8 +818,8 @@ impl Supervise {
         let _ = verdict.send(ok);
         let err = tail.and_then(|t| t.join().ok()).unwrap_or_default();
         if !ok {
-            eprintln!(
-                "sc-forge: git transport on repository {repo_id} ended ({reason}, {}): {}",
+            crate::log::warning!(
+                "git transport on repository {repo_id} ended ({reason}, {}): {}",
                 status.map_or_else(|e| e.to_string(), |s| s.to_string()),
                 String::from_utf8_lossy(&err).trim()
             );
@@ -964,7 +967,7 @@ pub async fn handle(app: Arc<App>, req: Req, body: Incoming, path: GitPath) -> R
         Ok(Err(LaunchError::Gone)) => return buffered(transport::repo_not_found()),
         Ok(Err(LaunchError::Stopped)) => return buffered(shutting_down()),
         Ok(Err(LaunchError::Io(e))) => {
-            eprintln!("sc-forge: could not start git: {e}");
+            crate::log::warning!("could not start git: {e}");
             return buffered(text(500, "internal server error\n"));
         }
         Err(e) if e.is_cancelled() => return buffered(shutting_down()),
@@ -1004,7 +1007,7 @@ pub async fn handle(app: Arc<App>, req: Req, body: Incoming, path: GitPath) -> R
             let why = abort.load(Ordering::SeqCst);
             let status = if why != 0 { why } else { 502 };
             if let Ok(Err(e)) = other {
-                eprintln!("sc-forge: git transport failed before answering: {e}");
+                crate::log::warning!("git transport failed before answering: {e}");
             }
             return buffered(text(status, "git transport failed\n"));
         }

crates/forge/src/cli.rs

@@ -104,6 +104,9 @@ struct ServeArgs {
     /// Wall-clock limit of the garbage collection of one repository, seconds.
     #[arg(long, default_value_t = 3_600)]
     gc_timeout: u64,
+    /// Seconds a SIGTERM waits for requests in flight before it ends the git processes still running.
+    #[arg(long, default_value_t = 30)]
+    shutdown_timeout: u64,
     /// Concurrent password verifications (Argon2 is memory-hard).
     #[arg(long, default_value_t = 4)]
     argon2_concurrency: usize,
@@ -278,6 +281,7 @@ fn run_serve(a: ServeArgs) -> Result<()> {
         ("--max-pack-size", a.max_pack_size),
         ("--max-request-bytes", a.max_request_bytes),
         ("--gc-timeout", a.gc_timeout),
+        ("--shutdown-timeout", a.shutdown_timeout),
     ] {
         if v == 0 {
             bail!("{flag} must be at least 1");
@@ -312,9 +316,9 @@ fn run_serve(a: ServeArgs) -> Result<()> {
         bail!("--trusted-proxy-header and --trusted-proxies must be given together");
     }
     if a.behind_proxy && cidrs.is_empty() {
-        eprintln!(
-            "sc-forge: warning: --behind-proxy without --trusted-proxy-header/--trusted-proxies: every client shares the proxy's address, \
-             so the per-source password-failure budget (30/min) applies to all of them together"
+        scl_forge::log::warning_line(
+            "warning: --behind-proxy without --trusted-proxy-header/--trusted-proxies: every client shares the proxy's address, \
+             so the per-source password-failure budget (30/min) applies to all of them together",
         );
     }
     let proxies = Proxies {
@@ -334,7 +338,7 @@ fn run_serve(a: ServeArgs) -> Result<()> {
         a.git_concurrency,
     )?);
     let git_version = scl_forge::gitproc::check_version(&git).map_err(anyhow::Error::msg)?;
-    eprintln!("sc-forge: {git_version}");
+    scl_forge::log::info_line(&git_version);
     scl_forge::hook::install(
         &std::fs::canonicalize(&a.data_dir)?,
         &std::env::current_exe().context("cannot find the sc-forge binary")?,
@@ -346,7 +350,7 @@ fn run_serve(a: ServeArgs) -> Result<()> {
     // Resolve whatever a crash left, before the listener is bound.
     let report = life.reconcile(&db::connect(&a.data_dir)?, scl_forge::timefmt::now())?;
     if report != scl_forge::lifecycle::Report::default() {
-        eprintln!("sc-forge: reconciled {report:?}");
+        scl_forge::log::info_line(&format!("reconciled {report:?}"));
     }
     // Pull request refs and rows agree again before anything is served.
     scl_forge::pulls::reconcile_all(&life, &db::connect(&a.data_dir)?, scl_forge::timefmt::now());
@@ -389,16 +393,25 @@ fn run_serve(a: ServeArgs) -> Result<()> {
         let listener = tokio::net::TcpListener::bind(a.listen)
             .await
             .with_context(|| format!("bind {}", a.listen))?;
-        eprintln!("sc-forge: listening on {}", listener.local_addr()?);
+        // Handlers first, announcement second: a SIGTERM that follows the "listening" line (a
+        // supervisor that stops a service the moment it is up) must drain, not kill the process.
+        use tokio::signal::unix::{signal, SignalKind};
+        let (mut term, mut int) = (
+            signal(SignalKind::terminate()).context("install the SIGTERM handler")?,
+            signal(SignalKind::interrupt()).context("install the SIGINT handler")?,
+        );
+        scl_forge::log::info_line(&format!("listening on {}", listener.local_addr()?));
         let stop = async {
-            use tokio::signal::unix::{signal, SignalKind};
-            let (mut term, mut int) = (
-                signal(SignalKind::terminate()).expect("signal"),
-                signal(SignalKind::interrupt()).expect("signal"),
-            );
             tokio::select! { _ = term.recv() => {}, _ = int.recv() => {} }
         };
-        serve(listener, app, limits, stop, Duration::from_secs(30)).await;
+        serve(
+            listener,
+            app,
+            limits,
+            stop,
+            Duration::from_secs(a.shutdown_timeout),
+        )
+        .await;
         Ok(())
     });
     // Queued password verifications run on blocking threads that the drain deadline

crates/forge/src/content_write.rs

@@ -442,17 +442,17 @@ fn run(ctx: &Ctx, a: &Access, row: &Repo, w: &Write) -> Resp {
                 )
             }
             Outcome::Fatal => {
-                eprintln!("sc-forge: update-ref {full} failed on attempt {attempt}: {failure}");
+                crate::log::warning!("update-ref {full} failed on attempt {attempt}: {failure}");
                 return ctx.app.internal(&"could not update the branch");
             }
         }
-        eprintln!("sc-forge: contents write to {full} lost round {attempt}: {failure}");
+        crate::log::warning!("contents write to {full} lost round {attempt}: {failure}");
     }
     if locked {
         // Three rounds against a ref lock that never cleared: nothing changed the branch. A killed
         // git can leave `refs/heads/<branch>.lock` behind; it blocks pushes as well.
-        eprintln!(
-            "sc-forge: refs/heads/{target_name} of repository {} stayed locked; if no git process is running, remove its .lock file",
+        crate::log::warning!(
+            "refs/heads/{target_name} of repository {} stayed locked; if no git process is running, remove its .lock file",
             a.repo.id
         );
         return error(&ctx.app.base, 503, "the branch is locked, try again later");
@@ -472,7 +472,7 @@ fn write_err(ctx: &Ctx, a: &Access, e: WriteErr) -> Resp {
     match e {
         WriteErr::Git(e) => read_err(ctx, a, e),
         WriteErr::Rewrite(m) => {
-            eprintln!("sc-forge: contents write could not rewrite a directory: {m}");
+            crate::log::warning!("contents write could not rewrite a directory: {m}");
             unprocessable(ctx, "a directory on the path cannot be rewritten")
         }
     }
@@ -481,8 +481,8 @@ fn write_err(ctx: &Ctx, a: &Access, e: WriteErr) -> Resp {
 /// The ref moved: stamp the repository (log-only, the write is already public) and answer.
 fn published(ctx: &Ctx, a: &Access, resp: Resp) -> Resp {
     if let Err(e) = repos::touch(ctx.conn, a.repo.id, now()) {
-        eprintln!(
-            "sc-forge: could not stamp repository {} after a contents write: {e}",
+        crate::log::warning!(
+            "could not stamp repository {} after a contents write: {e}",
             a.repo.id
         );
     }

crates/forge/src/db.rs

@@ -271,8 +271,8 @@ fn ensure_dir(data_dir: &Path) -> Result<(), DbError> {
     std::fs::create_dir_all(data_dir)?;
     let mode = std::fs::metadata(data_dir)?.permissions().mode() & 0o777;
     if mode != 0o700 {
-        eprintln!(
-            "sc-forge: tightening {} from {mode:o} to 700: it holds password and token hashes",
+        crate::log::warning!(
+            "tightening {} from {mode:o} to 700: it holds password and token hashes",
             data_dir.display()
         );
         std::fs::set_permissions(data_dir, std::fs::Permissions::from_mode(0o700))?;

crates/forge/src/lib.rs

@@ -23,6 +23,7 @@ pub mod hook;
 pub mod http;
 pub mod lifecycle;
 pub mod limits;
+pub mod log;
 pub mod maintenance;
 pub mod merges;
 pub mod names;

crates/forge/src/lifecycle.rs

@@ -332,8 +332,8 @@ impl Lifecycle {
                     Ok(()) => {
                         let _ = repos::delete_row(conn, row.id);
                     }
-                    Err(c) => eprintln!(
-                        "sc-forge: could not remove {} ({c}); the row stays `creating` for the next startup",
+                    Err(c) => crate::log::warning!(
+                        "could not remove {} ({c}); the row stays `creating` for the next startup",
                         self.repo_dir(row.id).display()
                     ),
                 }
@@ -450,8 +450,8 @@ impl Lifecycle {
         repos::delete_row(conn, repo.id)?;
         self.step("row-deleted")?;
         if let Err(e) = std::fs::remove_dir_all(&to) {
-            eprintln!(
-                "sc-forge: could not remove trash/{}.git ({e}); startup reconcile will",
+            crate::log::warning!(
+                "could not remove trash/{}.git ({e}); startup reconcile will",
                 repo.id
             );
         }
@@ -647,8 +647,8 @@ impl Lifecycle {
             };
             if !known {
                 let to = self.to_orphans(&p)?;
-                eprintln!(
-                    "sc-forge: {} has no repository row; moved to {}",
+                crate::log::warning!(
+                    "{} has no repository row; moved to {}",
                     p.display(),
                     to.display()
                 );
@@ -661,9 +661,11 @@ impl Lifecycle {
         for r in repos::list_in_state(conn, "live")? {
             let dir = self.repo_dir(r.id);
             if !dir.is_dir() {
-                eprintln!(
-                    "sc-forge: repository {}/{} (id {}) has no directory",
-                    r.owner, r.name, r.id
+                crate::log::warning!(
+                    "repository {}/{} (id {}) has no directory",
+                    r.owner,
+                    r.name,
+                    r.id
                 );
                 rep.missing_dirs += 1;
                 continue;
@@ -685,9 +687,11 @@ impl Lifecycle {
                 Ok(true) => rep.head_repaired += 1,
                 Ok(false) => {}
                 Err(e) => {
-                    eprintln!(
-                        "sc-forge: cannot repair HEAD of {}/{} (id {}): {e}",
-                        r.owner, r.name, r.id
+                    crate::log::warning!(
+                        "cannot repair HEAD of {}/{} (id {}): {e}",
+                        r.owner,
+                        r.name,
+                        r.id
                     );
                     rep.head_repair_failed += 1;
                 }