diff --git a/Cargo.lock b/Cargo.lock index f9660b53..04e98c16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2004,12 +2004,13 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.9.1", + "zlib-rs", ] [[package]] @@ -4252,6 +4253,16 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -4834,7 +4845,7 @@ dependencies = [ "crc32fast", "fdeflate", "flate2", - "miniz_oxide", + "miniz_oxide 0.8.9", ] [[package]] @@ -5547,6 +5558,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "flate2", "http-body-util", "hyper", "hyper-util", diff --git a/ROADMAP.md b/ROADMAP.md index 9245da97..790ec2c5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -702,8 +702,9 @@ a Gitea-compatible Git forge (bare repos over smart HTTP, a recorded subset of config. Design: `docs/superpowers/specs/2026-10-22-p38-forge-design.md`. Slices F0 (contract capture) through F6 (acceptance on the first deployment); F0, F1 (server -skeleton, users, organizations, tokens, `admin` CLI) and F2a (repositories API, lifecycle -and reconciler, collaborators, branches) are built. Hosted repos are +skeleton, users, organizations, tokens, `admin` CLI), F2a (repositories API, lifecycle +and reconciler, collaborators, branches) and F2b (smart-HTTP git transport via `git http-backend`, +`pre-receive` ref policy, admission control, threat-model pass #1) are built. Hosted repos are bare Git and get none of the `.sc/` features. Other follow-up work is tracked as GitHub issues and listed under Deferred diff --git a/crates/forge/Cargo.toml b/crates/forge/Cargo.toml index 10b00ce8..92d8b054 100644 --- a/crates/forge/Cargo.toml +++ b/crates/forge/Cargo.toml @@ -23,6 +23,7 @@ scl-crypto = { path = "../crypto" } # these out of other targets stops the Windows workspace check from compiling # bundled SQLite for an empty crate. [target.'cfg(unix)'.dependencies] +flate2 = "1.1.10" http-body-util = "0.1.5" hyper = { version = "1", features = ["http1", "server"] } hyper-util = { version = "0.1.21", features = ["tokio", "server"] } diff --git a/crates/forge/conformance/departures.json b/crates/forge/conformance/departures.json index f8b1a15a..c7d75b88 100644 --- a/crates/forge/conformance/departures.json +++ b/crates/forge/conformance/departures.json @@ -890,7 +890,7 @@ "expect": { "status": 404, "message_contains": [ - "not found" + "couldn't be found" ], "ignore_body": true }, @@ -904,7 +904,7 @@ "expect": { "status": 404, "message_contains": [ - "not found" + "couldn't be found" ], "ignore_body": true, "ignore_headers": [ @@ -950,7 +950,7 @@ "expect": { "status": 404, "message_contains": [ - "not found" + "couldn't be found" ] }, "reason": "the new name does not exist" @@ -963,7 +963,7 @@ "expect": { "status": 404, "message_contains": [ - "not found" + "couldn't be found" ], "ignore_body": true }, @@ -977,7 +977,7 @@ "expect": { "status": 404, "message_contains": [ - "not found" + "couldn't be found" ], "ignore_body": true }, @@ -1336,5 +1336,110 @@ ] }, "reason": "the health body names this server and reports only the checks it has (database), not Gitea's description or cache check; status and content type stay compared" + }, + { + "scenario": "C", + "exchange": 30, + "kind": "git", + "match": "push-read-collab", + "expect": { + "ok": false, + "message_contains": [ + "forbidden" + ] + }, + "reason": "sc-forge refuses a read-only collaborator's push at the advertisement with 403 Forbidden; Gitea advertises and then declines in pre-receive" + }, + { + "scenario": "J", + "exchange": 62, + "kind": "git", + "match": "delete-default-branch-by-push", + "expect": { + "ok": false, + "message_contains": [ + "default branch", + "pre-receive hook declined" + ] + }, + "reason": "sc-forge's hook words the refusal 'refusing to delete the default branch'; Gitea says 'gitea: branch main is the default branch and cannot be deleted'. Both end in 'pre-receive hook declined'" + }, + { + "scenario": "K", + "exchange": 34, + "kind": "http", + "match": "GET /acme/vis.git/info/refs?service=git-receive-pack", + "expect": { + "status": 403, + "message_contains": [ + "forbidden" + ], + "ignore_body": true, + "ignore_headers": [ + "content-type" + ] + }, + "reason": "sc-forge refuses a read-only collaborator's receive-pack advertisement with 403; Gitea advertises (200) and declines the push in pre-receive [ignore_body: the recorded 200 body is the advertisement]" + }, + { + "scenario": "V", + "exchange": 5, + "kind": "note", + "match": "v2_advert_first_bytes", + "expect": { + "value_prefix": "000eversion 2\\n" + }, + "reason": "stock git's protocol v2 advertisement has no '# service=' preamble; Gitea prepends one. Git clients accept both (the rest of the note carries the git version and is not pinned)" + }, + { + "scenario": "N", + "exchange": 13, + "kind": "http", + "expect": { + "status": 200, + "fields": { + "empty": true, + "default_branch": "main" + } + }, + "reason": "the row is the authority for the default branch: sc-forge never moves HEAD or the default on a push, so a default that was unborn stays unborn (Gitea's first-push heuristic is not copied)", + "match": "GET /api/v1/repos/acme/fresh1" + }, + { + "scenario": "N", + "exchange": 22, + "kind": "http", + "expect": { + "status": 200, + "fields": { + "empty": true, + "default_branch": "develop" + } + }, + "reason": "the row is the authority for the default branch: sc-forge never moves HEAD or the default on a push, so a default that was unborn stays unborn (Gitea's first-push heuristic is not copied)", + "match": "GET /api/v1/repos/acme/fresh3" + }, + { + "scenario": "N", + "exchange": 26, + "kind": "http", + "expect": { + "status": 200, + "fields": { + "default_branch": "develop" + } + }, + "reason": "the row is the authority for the default branch: sc-forge never moves HEAD or the default on a push, so a default that was unborn stays unborn (Gitea's first-push heuristic is not copied)", + "match": "GET /api/v1/repos/acme/fresh3" + }, + { + "scenario": "N", + "exchange": 16, + "kind": "note", + "match": "ls-remote-symref-trunk-first_stdout", + "expect": { + "value": "" + }, + "reason": "HEAD stays on the unborn default branch, so ls-remote --symref HEAD prints nothing (Gitea moved HEAD to the first pushed branch)" } ] diff --git a/crates/forge/conformance/forge-check.sh b/crates/forge/conformance/forge-check.sh index 4186d8c9..8a743f50 100755 --- a/crates/forge/conformance/forge-check.sh +++ b/crates/forge/conformance/forge-check.sh @@ -25,4 +25,4 @@ for _ in $(seq 50); do done python3 -I "$here/run.py" check --base-url "http://127.0.0.1:$port" --admin-token "$token" \ --fixtures "$here/fixtures/1.22.6" --departures "$here/departures.json" --pending "$here/slice-pending.json" \ - --only A,B,C,L,N,R "$@" || { echo "--- server log"; cat "$data/serve.log"; exit 1; } + --only A,B,C,J,K,L,M,N,R,V "$@" || { echo "--- server log"; cat "$data/serve.log"; exit 1; } diff --git a/crates/forge/conformance/run.py b/crates/forge/conformance/run.py index 086bb797..75c6a101 100644 --- a/crates/forge/conformance/run.py +++ b/crates/forge/conformance/run.py @@ -369,7 +369,7 @@ def dep_indexes(spec, expected): EXPECT_KEYS = { "http": {"status", "message_contains", "fields", "ignore_fields", "ignore_headers", "ignore_body", "skip"}, "git": {"ok", "message_contains", "skip"}, - "note": {"value", "skip"}, + "note": {"value", "value_prefix", "skip"}, } @@ -461,7 +461,13 @@ def apply_override(exp, act, ov, bij, match=None): if act["kind"] == "note": if "value" in e and not eq_val(e["value"], act["value"], bij): d.append(("fail", f"unexpected: note {act['key']} {e['value']!r} expected, got {act['value']!r}")) + if "value_prefix" in e and not (isinstance(act["value"], str) and act["value"].startswith(e["value_prefix"])): + d.append(("fail", f"unexpected: note {act['key']} should start with {e['value_prefix']!r}, got {act['value']!r}")) elif act["kind"] == "git": + # a departure replaces the outcome, never the command: the step must still run the recorded line + bij.new_exchange() + if exp.get("cmd") is not None and not eq_str(exp["cmd"], act["cmd"], bij, [], ""): + return [("fail", f"git command {exp['cmd']!r} expected, got {act['cmd']!r}")] if "ok" in e and act["ok"] != e["ok"]: d.append(("fail", f"unexpected: git ok={e['ok']} expected, got {act['ok']}: {act['output'][:160]!r}")) txt = act["output"].lower() @@ -525,6 +531,7 @@ def __init__(self, base, admin_token, mode): self.cursor = 0 self.diffs = [] # (scenario, index, label, [(level, text)]) self.overrides = {} # fixture index -> departure entry, check mode only + self.defers = {} # fixture index -> exchange identity: runs, but a mismatch waits for a later slice self.verbose = False self.base_ready = False self.first_read = None @@ -560,7 +567,7 @@ def scrub(self, s): return re.sub(r"\b%s\b" % re.escape(self.admin_user), "", s) def ns(self, s): - s = re.sub(r"(://)[^/@\s\"':]*:[^/@\s\"']*@", r"\1@", self.scrub(s)) # user:password@ only (ssh://git@ stays) + s = re.sub(r"https?(://)[^/@\s\"':?#]*:[^/@\s\"'?#]*@", r"http\1@", self.scrub(s)) # user:password@ only (ssh://git@ stays); https checks like http # pkt-line adverts glue a 4-hex length in front of the id ("0146 HEAD"), so \b misses it s = re.sub(r"[0-9a-f]{40}(?= (?:HEAD|refs/|capabilities))", lambda m: self.sha_ph(m.group(0)), s) s = SHA_RE.sub(lambda m: self.sha_ph(m.group(0)), s) @@ -620,6 +627,14 @@ def push_ex(self, ex): ds = apply_override(exp, ex, self.overrides[i], self.bij) else: ds = cmp_exchange(exp, ex, self.bij) + if i in self.defers: + bad = [x for x in ds if x[0] == "fail"] + if ex_ident(exp) != ex_ident(ex) or exp["kind"] != ex["kind"]: + # a deferral waits for a later slice's *answer*; the call itself stays pinned + ds = [("fail", f"deferred exchange is {ex_ident(exp)!r} in the fixture but the scenario made {ex_ident(ex)!r}")] + else: + ds = ([("honoured", "deferred: this route belongs to a later slice")] if bad else + [("fail", "deferred exchange now passes: remove it from the pending file")]) self.diffs.append((i, ex_label(ex), ds)) def note(self, key, value, strict=True): @@ -938,7 +953,7 @@ def run_all(c, only, out=None, fixtures=None, departures=(), pending=None): c.ex, c.cursor, c.diffs = [], 0, [] c.reset_scenario() c.expected, skip, stale, deferred, ndeferred, stop = [], None, [], None, 0, None - c.overrides, c.web_skipped = {}, 0 + c.overrides, c.web_skipped, c.defers = {}, 0, {} if c.mode == "check" and selected: fp = os.path.join(fixtures, fname) if not os.path.exists(fp): @@ -973,8 +988,17 @@ def run_all(c, only, out=None, fixtures=None, departures=(), pending=None): f"cannot match the recorded one, so it needs ignore_body with a reason") continue c.overrides[i] = dict(dep, match=matches[k] or got) - if sid in pending: - deferred, stop = pending[sid] if isinstance(pending[sid], list) else (pending[sid], None) + for i, ident in (pending.get(sid) or {}).get("defer", []) if isinstance(pending.get(sid), dict) else []: + if not 0 <= i < len(c.expected) or ex_ident(c.expected[i]) != ident: + stale.append(f"stale defer #{i}: {fname} has {ex_ident(c.expected[i])!r} there, the pending file says {ident!r}" + if 0 <= i < len(c.expected) else f"stale defer #{i}: {fname} has no such exchange") + else: + c.defers[i] = ident + spec = pending.get(sid) + if isinstance(spec, dict): # {"stop": [N, text], "defer": [[index, "METHOD /path"], ...]} + spec = spec.get("stop") + if spec is not None: + deferred, stop = spec if isinstance(spec, list) else (spec, None) ndeferred = max(0, len(c.expected) - deferred) if not 0 <= deferred < len(c.expected): # nothing is deferred: the entry is stale (the scenario is whole) and must not hide an error diff --git a/crates/forge/conformance/slice-pending.json b/crates/forge/conformance/slice-pending.json index faa170a3..db920199 100644 --- a/crates/forge/conformance/slice-pending.json +++ b/crates/forge/conformance/slice-pending.json @@ -1,5 +1,6 @@ { "B": [25, "GET /api/v1/repos/alice/scoped/contents/README.md"], - "C": [27, "git clone"], - "N": [0, "git clone"] + "J": {"defer": [[56, "GET /api/v1/repos/acme/smart/tags"], [57, "GET /api/v1/repos/acme/smart/git/refs/tags"], [59, "GET /api/v1/repos/acme/smart/tags"]]}, + "K": {"defer": [[21, "GET /api/v1/repos/acme/vis/contents"], [26, "GET /api/v1/repos/acme/ghost/contents"], [31, "GET /api/v1/repos/acme/vis/contents"], [36, "GET /api/v1/repos/acme/ghost/contents"], [55, "GET /api/v1/orgs/acme/repos"], [56, "GET /api/v1/orgs/acme/repos?limit=50"]]}, + "N": {"defer": [[6, "GET /api/v1/repos/acme/dflt/contents"]]} } diff --git a/crates/forge/src/cgi.rs b/crates/forge/src/cgi.rs new file mode 100644 index 00000000..8f88901d --- /dev/null +++ b/crates/forge/src/cgi.rs @@ -0,0 +1,1019 @@ +//! Serving a git request: admission, the `git http-backend` child, and the streams around it. +//! +//! One transport is a child process in its own group, owned by exactly one supervisor thread +//! (it signals the group while the leader is still an unreaped zombie, so the group id cannot have +//! been reused), plus three small threads: stdin writer, stdout reader and a stderr tail. The +//! request body reaches the child through a bounded channel fed by an async pump that enforces the +//! size cap, inflates gzip under a cap on *inflated* bytes and applies the idle deadline; the +//! child's output reaches the client through a bounded channel read by a streaming response body. +//! Nothing is buffered beyond those channels, so a clone or push costs a bounded amount of memory. +//! +//! What may be asked of git is decided in `transport.rs` before any of this runs. + +use crate::http::Req; +use crate::registry::{Cancel, CancelOnDrop, Registration}; +use crate::repos; +use crate::routes::App; +use crate::transport::{self, GitPath, Head, Held, Plan, Reserve, Service}; +use http_body_util::combinators::BoxBody; +use http_body_util::BodyExt; +use hyper::body::{Body, Bytes, Frame, Incoming}; +use hyper::Response; +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::os::unix::process::CommandExt; +use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command, Stdio}; +use std::sync::atomic::{AtomicU16, Ordering}; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; +use tokio::sync::{mpsc, oneshot, OwnedSemaphorePermit, Semaphore}; + +/// The response body type: a buffered answer or a stream from the child. +pub type RespBody = BoxBody; + +/// Transport limits (flags of `sc-forge serve`). +#[derive(Clone, Debug)] +pub struct Cfg { + /// Transports in flight, all repositories. + pub max_transports: usize, + /// Transports in flight on one repository. + pub per_repo: usize, + /// How long a request waits for both slots before 503. + pub admission_wait: Duration, + /// Wall-clock limit of one transport. + pub timeout: Duration, + /// How long the server waits on a stalled client (no request bytes while the child wants some, + /// or no room in the response while the child has some). + pub idle: Duration, + /// Largest push, bytes (also `receive.maxInputSize`). + pub max_pack: u64, + /// Largest upload-pack request, compressed and inflated. + pub max_request: u64, + /// Disk that must stay free beyond what pushes in flight have reserved. + pub min_free: u64, + /// Initial reservation of a push whose size is not announced. + pub push_reserve: u64, + /// `RLIMIT_CPU` of the child, seconds. + pub cpu_secs: u64, + /// `RLIMIT_DATA` of the child on Linux, bytes (0 = none). + pub git_memory: u64, +} + +impl Default for Cfg { + fn default() -> Cfg { + Cfg { + max_transports: 64, + per_repo: 16, + admission_wait: Duration::from_secs(30), + timeout: Duration::from_secs(1800), + idle: Duration::from_secs(120), + max_pack: 2 << 30, + max_request: 16 << 20, + min_free: 1 << 30, + push_reserve: 256 << 20, + cpu_secs: 900, + git_memory: 8 << 30, + } + } +} + +/// Reservation grows in steps of this many bytes as a push is forwarded. +const GROW_STEP: u64 = 64 << 20; + +/// Admission state: the two slot pools and the disk reservation. +pub struct Transports { + pub cfg: Cfg, + global: Arc, + per_repo: Mutex>>, + reserve: Arc, +} + +/// Slots held for one transport; returned on drop. +pub struct Permits { + _repo: OwnedSemaphorePermit, + _global: OwnedSemaphorePermit, +} + +impl Transports { + pub fn new(cfg: Cfg) -> Transports { + Transports { + global: Arc::new(Semaphore::new(cfg.max_transports.max(1))), + per_repo: Mutex::default(), + reserve: Arc::new(Reserve::default()), + cfg, + } + } + + fn repo_slots(&self, repo: i64) -> Arc { + let mut m = self.per_repo.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 256 { + // Forget pools nobody uses (all permits home, no waiter holds the Arc). + let full = self.cfg.per_repo.max(1); + m.retain(|_, s| s.available_permits() < full || Arc::strong_count(s) > 1); + } + m.entry(repo) + .or_insert_with(|| Arc::new(Semaphore::new(self.cfg.per_repo.max(1)))) + .clone() + } + + /// Wait up to the admission deadline for a repository slot, then a global one (this order, so + /// a flood on one repository queues behind its own limit instead of holding global slots + /// while it waits). `None` means busy: the caller answers 503. + pub async fn admit(&self, repo: i64) -> Option { + let repo_pool = self.repo_slots(repo); + let global = self.global.clone(); + tokio::time::timeout(self.cfg.admission_wait, async move { + let r = repo_pool.acquire_owned().await.ok()?; + let g = global.acquire_owned().await.ok()?; + Some(Permits { + _repo: r, + _global: g, + }) + }) + .await + .ok() + .flatten() + } + + /// Slots currently free (tests). + pub fn free_slots(&self) -> usize { + self.global.available_permits() + } +} + +// ---- the response body ---- + +/// A response body fed by the stdout reader thread. Dropping it (the client went away, or the +/// response finished) cancels the transport. +struct ChannelBody { + rx: mpsc::Receiver>, + _guard: CancelOnDrop, +} + +impl Body for ChannelBody { + type Data = Bytes; + type Error = std::io::Error; + + fn poll_frame( + mut self: std::pin::Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, std::io::Error>>> { + // An empty chunk is the explicit end; a channel that closes without one means the + // transport was cut short, which must reach the client as an error, not a clean end. + self.rx.poll_recv(cx).map(|o| match o { + Some(Ok(b)) if b.is_empty() => None, + Some(r) => Some(r.map(Frame::data)), + None => Some(Err(std::io::Error::other("transport aborted"))), + }) + } +} + +fn buffered(r: crate::http::Resp) -> Response { + let mut b = Response::builder().status(r.status); + for (k, v) in r.headers { + b = b.header(k, v); + } + b.body( + http_body_util::Full::new(Bytes::from(r.body)) + .map_err(|e| match e {}) + .boxed(), + ) + .unwrap_or_else(|_| fallback(500)) +} + +fn fallback(status: u16) -> Response { + let mut r = Response::new( + http_body_util::Full::new(Bytes::from_static(b"internal server error\n")) + .map_err(|e| match e {}) + .boxed(), + ); + *r.status_mut() = + hyper::StatusCode::from_u16(status).unwrap_or(hyper::StatusCode::INTERNAL_SERVER_ERROR); + r +} + +fn text(status: u16, body: &str) -> crate::http::Resp { + crate::http::Resp { + status, + headers: vec![("content-type", "text/plain;charset=utf-8".into())], + body: body.as_bytes().to_vec(), + delay: Duration::ZERO, + } +} + +// ---- the request pump ---- + +/// Inflates into a buffer and refuses to grow past `cap` bytes in total. +struct Sink { + buf: Vec, + total: u64, + cap: u64, + over: bool, +} + +impl Write for Sink { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.total += b.len() as u64; + if self.total > self.cap { + self.over = true; + return Err(std::io::Error::other("inflated request too large")); + } + self.buf.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Why the pump stopped the transport (the status the client gets if no response had started). +const ABORT_TIMEOUT: u16 = 408; +const ABORT_BAD: u16 = 400; +const ABORT_TOO_LARGE: u16 = 413; +const ABORT_NO_SPACE: u16 = 507; + +/// Read a gzip request body completely and inflate it, before any process exists: a bad or +/// oversized body is then a plain 400/413, never a race with the response head `git http-backend` +/// writes before it reads its input. `cap` bounds both the compressed and the inflated size. +async fn inflate_body(body: Incoming, cap: u64, idle: Duration) -> Result, u16> { + // The per-frame idle deadline alone lets a client that trickles a byte per interval hold the + // admission slots for as long as it likes; the whole read gets a deadline too. + tokio::time::timeout(PRELOAD_DEADLINE, inflate_frames(body, cap, idle)) + .await + .unwrap_or(Err(ABORT_TIMEOUT)) +} + +/// Longest a gzip request body may take to arrive in full. +const PRELOAD_DEADLINE: Duration = Duration::from_secs(300); + +async fn inflate_frames(mut body: Incoming, cap: u64, idle: Duration) -> Result, u16> { + let mut dec = flate2::write::GzDecoder::new(Sink { + buf: Vec::new(), + total: 0, + cap, + over: false, + }); + let mut seen: u64 = 0; + loop { + let frame = match tokio::time::timeout(idle, body.frame()).await { + Err(_) => return Err(ABORT_TIMEOUT), + Ok(None) => break, + Ok(Some(Err(_))) => return Err(ABORT_BAD), + Ok(Some(Ok(f))) => f, + }; + let Ok(data) = frame.into_data() else { + continue; + }; + seen += data.len() as u64; + if seen > cap { + return Err(ABORT_TOO_LARGE); + } + if dec.write_all(&data).is_err() { + return Err(if dec.get_ref().over { + ABORT_TOO_LARGE + } else { + ABORT_BAD + }); + } + } + match dec.finish() { + Ok(sink) => Ok(sink.buf), + Err(_) => Err(ABORT_BAD), + } +} + +struct Pump { + /// The client's request body, streamed; `None` when `preload` holds all of it. + body: Option, + preload: Vec, + tx: mpsc::Sender, + /// Abort: delete, timeout, the client went away, shutdown. + cancel: Cancel, + /// The transport ended on its own (set by the supervisor last). + done: Cancel, + abort: Arc, + idle: Duration, + cap: u64, + reservation: Option, +} + +/// A push's disk reservation as the pump grows it. +struct Reservation { + held: Arc>, + reserved: u64, + data_dir: std::path::PathBuf, + floor: u64, +} + +impl Pump { + fn stop(&self, status: u16) { + self.abort.store(status, Ordering::SeqCst); + self.cancel.cancel(); + } + + async fn stopped(&self) { + tokio::select! { () = self.cancel.cancelled() => {}, () = self.done.cancelled() => {} } + } + + /// Hand `chunk` to the child; `false` when the pump must stop. + async fn send(&mut self, chunk: Bytes, forwarded: u64) -> bool { + if let Some(r) = &mut self.reservation { + while forwarded > r.reserved { + let free = transport::free_bytes(&r.data_dir).unwrap_or(0); + let ok = r.held.lock().unwrap_or_else(|e| e.into_inner()).grow( + // free space already lost this push's own bytes, which `held` still counts + free.saturating_add(forwarded.min(r.reserved)), + r.floor, + GROW_STEP, + ); + if !ok { + eprintln!("sc-forge: push aborted: the disk reserve cannot cover it"); + self.stop(ABORT_NO_SPACE); + return false; + } + r.reserved += GROW_STEP; + } + } + tokio::select! { + () = self.stopped() => false, + r = self.tx.send_timeout(chunk, self.idle) => match r { + Ok(()) => true, + Err(mpsc::error::SendTimeoutError::Timeout(_)) => { self.stop(ABORT_TIMEOUT); false } + // The child stopped reading (it answered without the rest): not an error. + Err(mpsc::error::SendTimeoutError::Closed(_)) => false, + }, + } + } + + async fn run(mut self) { + let mut forwarded: u64 = 0; + for p in pieces_of(std::mem::take(&mut self.preload)) { + forwarded += p.len() as u64; + if !self.send(p, forwarded).await { + return; + } + } + let Some(mut body) = self.body.take() else { + return; // dropping the sender ends the child's stdin + }; + loop { + let frame = tokio::select! { + () = self.stopped() => return, + f = tokio::time::timeout(self.idle, body.frame()) => f, + }; + let frame = match frame { + Err(_) => return self.stop(ABORT_TIMEOUT), + Ok(None) => return, + // The client's connection broke mid-request: end the transport. + Ok(Some(Err(_))) => return self.cancel.cancel(), + Ok(Some(Ok(f))) => f, + }; + let Ok(data) = frame.into_data() else { + continue; + }; + forwarded += data.len() as u64; + if forwarded > self.cap { + return self.stop(ABORT_TOO_LARGE); + } + if !self.send(data, forwarded).await { + return; + } + } + } +} + +fn pieces_of(buf: Vec) -> Vec { + buf.chunks(64 * 1024).map(Bytes::copy_from_slice).collect() +} + +// ---- the child ---- + +/// What a started transport hands back to the request task. +struct Started { + head: oneshot::Receiver>, + body: mpsc::Receiver>, + stdin: mpsc::Sender, + cancel: Cancel, + done: Cancel, +} + +/// Everything `launch` needs. +struct Launch { + app: Arc, + plan: Plan, + content_length: Option, + content_type: Option, + git_protocol: Option, + remote: std::net::IpAddr, + permits: Permits, + held: Option>>, + rt: tokio::runtime::Handle, +} + +enum LaunchError { + /// The repository is gone or closing (delete): answer as for a missing repository. + Gone, + /// The server is shutting down: retry elsewhere or later. + Stopped, + Io(std::io::Error), +} + +impl From for LaunchError { + fn from(e: std::io::Error) -> Self { + LaunchError::Io(e) + } +} + +/// The CGI variables and git config of one transport, on top of the scrubbed base environment. +fn environment(l: &Launch, marker: &std::path::Path) -> Vec<(String, String)> { + let cfg = &l.app.transports.cfg; + let git = &l.app.life.git; + let extra = [ + ("transfer.hideRefs", "refs/forge/".to_string()), + ("uploadpack.allowFilter", "true".into()), + ("uploadpack.allowAnySHA1InWant", "true".into()), + ("uploadpackfilter.allow", "false".into()), + ("uploadpackfilter.blob:none.allow", "true".into()), + ("receive.maxInputSize", cfg.max_pack.to_string()), + ("receive.autogc", "false".into()), + ("receive.advertisePushOptions", "true".into()), + ]; + let mut env = git.env_with(None, &extra); + let id = l.plan.access.repo.id; + let mut var = |k: &str, v: String| env.push((k.to_string(), v)); + var( + "GIT_PROJECT_ROOT", + git.data_dir().join("repos").display().to_string(), + ); + var("GIT_HTTP_EXPORT_ALL", "1".into()); + var( + "PATH_INFO", + format!("/{id}.git{}", l.plan.path.sub.path_info()), + ); + var("REQUEST_METHOD", l.plan.path.sub.method().to_string()); + var( + "QUERY_STRING", + match l.plan.path.sub { + transport::Sub::InfoRefs => l + .plan + .service + .map(|s| format!("service={}", s.name())) + .unwrap_or_default(), + _ => String::new(), + }, + ); + if let Some(ct) = &l.content_type { + var("CONTENT_TYPE", ct.clone()); + } + if let Some(n) = l.content_length { + var("CONTENT_LENGTH", n.to_string()); + } + if let Some(p) = &l.git_protocol { + var("HTTP_GIT_PROTOCOL", p.clone()); + } + if l.plan.may_write { + var("REMOTE_USER", l.plan.user.login.clone()); + } + var("REMOTE_ADDR", l.remote.to_string()); + var("GIT_HTTP_MAX_REQUEST_BUFFER", cfg.max_request.to_string()); + var("FORGE_DATA_DIR", git.data_dir().display().to_string()); + var("FORGE_REPO_ID", id.to_string()); + var("FORGE_USER_ID", l.plan.user.id.to_string()); + var("FORGE_ACCEPT_MARKER", marker.display().to_string()); + env +} + +fn launch(l: Launch) -> Result { + let repo_id = l.plan.access.repo.id; + let cfg = l.app.transports.cfg.clone(); + let conn = crate::db::connect(&l.app.data_dir).map_err(std::io::Error::other)?; + let reg = l + .app + .life + .registry + .register( + repo_id, + || matches!(repos::find_by_id(&conn, repo_id), Ok(Some(r)) if r.state == "live"), + ) + .ok_or_else(|| { + if l.app.life.registry.is_stopped() { + LaunchError::Stopped + } else { + LaunchError::Gone + } + })?; + let cancel = reg.cancel.clone(); + + let git = &l.app.life.git; + static MARKERS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let marker = git.data_dir().join("tmp").join(format!( + "accept-{}-{}", + std::process::id(), + MARKERS.fetch_add(1, Ordering::SeqCst) + )); + let (cpu, fsize, mem) = ( + cfg.cpu_secs, + cfg.max_pack.saturating_add(256 << 20), + cfg.git_memory, + ); + let mut cmd = Command::new(git.bin()); + cmd.arg("http-backend") + .env_clear() + .envs(environment(&l, &marker)) + .current_dir(git.data_dir()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + // SAFETY: the closure runs between fork and exec and only calls setrlimit (async-signal-safe). + unsafe { + cmd.pre_exec(move || { + // Heap and anonymous mappings (file-backed pack mappings are not counted): a crafted + // pack declaring a huge compressed object cannot take the host's memory. Linux only, + // where RLIMIT_DATA covers mmap since 4.7; macOS does not enforce it usefully. + // (A fixed array: nothing may allocate between fork and exec.) + let data = if cfg!(target_os = "linux") { mem } else { 0 }; + let limits = [ + (libc::RLIMIT_CPU, cpu), + (libc::RLIMIT_FSIZE, fsize), + (libc::RLIMIT_CORE, 0), + (libc::RLIMIT_DATA, data), + ]; + for (res, v) in limits { + if res == libc::RLIMIT_DATA && v == 0 { + continue; + } + let lim = libc::rlimit { + rlim_cur: v as libc::rlim_t, + rlim_max: v as libc::rlim_t, + }; + // A refused limit (a lower hard limit already in force) is not fatal. + let _ = libc::setrlimit(res, &lim); + } + Ok(()) + }); + } + // A marker left by a killed server must not pass for this push's acceptance. + let _ = std::fs::remove_file(&marker); + if cancel.is_set() { + return Err(if l.app.life.registry.is_stopped() { + LaunchError::Stopped + } else { + LaunchError::Gone + }); + } + let mut child = cmd.spawn()?; + let pgid = child.id() as i32; + let (stdin, stdout, stderr) = ( + child.stdin.take().expect("piped"), + child.stdout.take().expect("piped"), + child.stderr.take().expect("piped"), + ); + + let (stdin_tx, stdin_rx) = mpsc::channel::(4); + let (body_tx, body_rx) = mpsc::channel::>(4); + let (head_tx, head_rx) = oneshot::channel(); + let (verdict_tx, verdict_rx) = std::sync::mpsc::channel::(); + + let done = Cancel::default(); + let child_slot = Arc::new(Mutex::new(Some(child))); + let sup = Supervise { + child: child_slot.clone(), + done: done.clone(), + stderr, + reg, + _permits: l.permits, + _held: l.held, + verdict: verdict_tx, + timeout: cfg.timeout, + app: l.app.clone(), + repo_id, + marker, + // Only the push itself stamps the repository, not the advertisement that precedes it. + receive: l.plan.path.sub == transport::Sub::ReceivePack, + }; + if let Err(e) = std::thread::Builder::new() + .name("forge-supervise".into()) + .spawn(move || sup.run()) + { + // The supervisor never started, so the child is still ours: kill the group, then reap. + // SAFETY: killpg only sends a signal to the group this call created, whose leader is unreaped. + unsafe { libc::killpg(pgid, libc::SIGKILL) }; + if let Some(mut c) = child_slot.lock().unwrap_or_else(|e| e.into_inner()).take() { + let _ = c.wait(); + } + return Err(e.into()); + } + let stall = cfg.idle; + let rt = l.rt.clone(); + let spawned = std::thread::Builder::new() + .name("forge-stdin".into()) + .spawn(move || write_in(stdin, stdin_rx)) + .and_then(|_| { + let c = cancel.clone(); + std::thread::Builder::new() + .name("forge-stdout".into()) + .spawn(move || read_out(stdout, head_tx, body_tx, verdict_rx, stall, c, rt)) + }); + if let Err(e) = spawned { + cancel.cancel(); // the supervisor kills the group + return Err(e.into()); + } + Ok(Started { + head: head_rx, + body: body_rx, + stdin: stdin_tx, + cancel, + done, + }) +} + +/// Copy the request to the child's stdin; dropping the handle at the end is its EOF. +fn write_in(mut stdin: ChildStdin, mut rx: mpsc::Receiver) { + while let Some(b) = rx.blocking_recv() { + if stdin.write_all(&b).is_err() { + break; // the child exited or closed its input + } + } +} + +/// Parse the CGI header block, report it, then stream the rest to the client. After the child +/// is reaped the supervisor's verdict decides whether the stream ends cleanly or with an error +/// (which makes the server drop the connection instead of terminating the chunked body). +fn read_out( + mut out: ChildStdout, + head_tx: oneshot::Sender>, + body_tx: mpsc::Sender>, + verdict: std::sync::mpsc::Receiver, + stall: Duration, + cancel: Cancel, + rt: tokio::runtime::Handle, +) { + // Cancellation-aware: a reader stuck on a client that does not read must not outlive the + // transport it belongs to. + let forward = |item: std::io::Result| -> bool { + rt.block_on(async { + tokio::select! { + () = cancel.cancelled() => false, + r = body_tx.send_timeout(item, stall) => r.is_ok(), + } + }) + }; + let mut buf = Vec::new(); + let mut chunk = vec![0u8; 32 * 1024]; + let (head, rest) = loop { + match out.read(&mut chunk) { + Ok(0) | Err(_) => { + let _ = head_tx.send(Err("git produced no response")); + return; + } + Ok(n) => { + buf.extend_from_slice(&chunk[..n]); + match transport::parse_head(&buf) { + Ok(Some((h, used))) => break (h, buf.split_off(used)), + Ok(None) => {} + Err(e) => { + let _ = head_tx.send(Err(e)); + cancel.cancel(); + return; + } + } + } + } + }; + if head_tx.send(Ok(head)).is_err() { + return cancel.cancel(); // the request task is gone + } + if !rest.is_empty() && !forward(Ok(Bytes::from(rest))) { + return cancel.cancel(); + } + loop { + match out.read(&mut chunk) { + Ok(0) => break, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + // A read error is not an end of file: the client must see the cut, whatever git's + // exit status turns out to be. + let _ = forward(Err(e)); + return cancel.cancel(); + } + Ok(n) => { + if !forward(Ok(Bytes::copy_from_slice(&chunk[..n]))) { + return cancel.cancel(); // client gone or stalled + } + } + } + } + // Only a clean finish ends the body with the explicit end marker; every other way out + // (error frame here, or the channel simply closing) reaches the client as an error. + if verdict.recv().unwrap_or(false) { + let _ = forward(Ok(Bytes::new())); + } else { + let _ = forward(Err(std::io::Error::other("git did not finish"))); + } +} + +struct Supervise { + child: Arc>>, + done: Cancel, + stderr: ChildStderr, + reg: Registration, + _permits: Permits, + _held: Option>>, + verdict: std::sync::mpsc::Sender, + timeout: Duration, + app: Arc, + repo_id: i64, + /// Written by the pre-receive hook when it let the push through. + marker: std::path::PathBuf, + receive: bool, +} + +impl Supervise { + fn run(self) { + let Supervise { + child, + done, + stderr, + reg, + verdict, + timeout, + app, + repo_id, + marker, + receive, + .. + } = self; + let Some(mut child) = child.lock().unwrap_or_else(|e| e.into_inner()).take() else { + return; + }; + let pgid = child.id() as i32; + let cancel = reg.cancel.clone(); + let tail = std::thread::Builder::new() + .name("forge-stderr".into()) + .spawn(move || stderr_tail(stderr)) + .ok(); + let end = Instant::now() + timeout; + let mut nap = Duration::from_millis(2); + // Detect exit without reaping: while the leader is a zombie its pid, and so the group id, + // cannot be reused. Signal the group first, reap after. + let reason = loop { + match crate::gitproc::exited_unreaped(pgid) { + Ok(true) => break "exited", + Ok(false) => {} + Err(_) => break "lost", + } + if cancel.is_set() { + break "cancelled"; + } + if Instant::now() >= end { + break "timed out"; + } + std::thread::sleep(nap); + nap = (nap * 2).min(Duration::from_millis(50)); + }; + // Ask first when we are the ones ending it: git removes its lock files and quarantine + // directories from its signal handlers, SIGKILL would leave them behind. Then, whatever the + // reason, end the whole group (a hook's background child may outlive the leader). + // SAFETY: killpg only sends a signal; the leader is unreaped, so the group id is ours. + if reason != "exited" { + unsafe { libc::killpg(pgid, libc::SIGTERM) }; + let until = Instant::now() + Duration::from_secs(1); + // Until the leader is gone and stderr has closed (every group member that inherited it + // has exited, so git's children have finished their own cleanup), at most one second. + while Instant::now() < until + && (matches!(crate::gitproc::exited_unreaped(pgid), Ok(false)) + || tail.as_ref().is_some_and(|t| !t.is_finished())) + { + std::thread::sleep(Duration::from_millis(5)); + } + } + unsafe { libc::killpg(pgid, libc::SIGKILL) }; + let status = child.wait(); + let ok = reason == "exited" && status.as_ref().is_ok_and(|s| s.success()); + let _ = verdict.send(ok); + let err = tail.and_then(|t| t.join().ok()).unwrap_or_default(); + if !ok { + eprintln!( + "sc-forge: git transport on repository {repo_id} ended ({reason}, {}): {}", + status.map_or_else(|e| e.to_string(), |s| s.to_string()), + String::from_utf8_lossy(&err).trim() + ); + } else if receive && marker.exists() { + if let Ok(conn) = crate::db::connect(&app.data_dir) { + let _ = repos::touch(&conn, repo_id, crate::timefmt::now()); + } + } + let _ = std::fs::remove_file(&marker); + // Wake the request pump, which may still be waiting on the client. (Not `cancel`: that + // aborts the response reader, which may still be forwarding the last bytes.) + done.cancel(); + } +} + +/// The last few KiB of the child's stderr (drained to EOF so it never blocks on a full pipe). +fn stderr_tail(mut e: ChildStderr) -> Vec { + const KEEP: usize = 4096; + let mut tail: Vec = Vec::new(); + let mut buf = [0u8; 4096]; + while let Ok(n) = e.read(&mut buf) { + if n == 0 { + break; + } + tail.extend_from_slice(&buf[..n]); + if tail.len() > KEEP { + tail.drain(..tail.len() - KEEP); + } + } + tail +} + +// ---- the request task ---- + +/// Serve one git request: the caller has matched the path to `path`. +pub async fn handle(app: Arc, req: Req, body: Incoming, path: GitPath) -> Response { + let req = Arc::new(req); + let plan = { + let (app, req) = (app.clone(), req.clone()); + match tokio::task::spawn_blocking(move || transport::authorize(&app, &req, path)).await { + Ok(Ok(p)) => p, + Ok(Err(r)) => return buffered(r), + Err(_) => return buffered(text(500, "internal server error\n")), + } + }; + let t = &app.transports; + let service = plan.service; + let is_receive = service == Some(Service::Receive); + let has_body = plan.path.sub.method() == "POST"; + let is_rpc = has_body; + let content_length = match req + .header("content-length") + .map(|v| v.trim().parse::()) + { + None => None, + Some(Ok(n)) => Some(n), + Some(Err(_)) => return buffered(text(400, "Bad Request\n")), + }; + let gzip = match req + .header("content-encoding") + .map(|v| v.trim().to_ascii_lowercase()) + { + None => false, + Some(e) if e == "identity" => false, + Some(e) if (e == "gzip" || e == "x-gzip") && service == Some(Service::Upload) => true, + Some(_) => return buffered(text(415, "Unsupported Media Type\n")), + }; + let cap = if is_receive { + t.cfg.max_pack + } else { + t.cfg.max_request + }; + if has_body && content_length.is_some_and(|n| n > cap) { + return buffered(text(413, "Request Entity Too Large\n")); + } + let repo_id = plan.access.repo.id; + let Some(permits) = t.admit(repo_id).await else { + return buffered(text(503, "Service Unavailable\n").with("retry-after", "5".into())); + }; + // A gzip request is read and inflated whole before anything starts, so its 400/413 is a plain + // answer and a bomb never gets a process (git itself would buffer an upload-pack request anyway). + let mut body = Some(body); + let mut preload = Vec::new(); + if gzip { + match inflate_body(body.take().expect("present"), cap, t.cfg.idle).await { + Ok(v) => preload = v, + Err(status) => { + let msg = match status { + ABORT_TOO_LARGE => "Request Entity Too Large\n", + ABORT_TIMEOUT => "Request Timeout\n", + _ => "Bad Request\n", + }; + return buffered(text(status, msg)); + } + } + } + let content_length = if gzip { + Some(preload.len() as u64) + } else { + content_length + }; + let mut reservation = None; + let mut held = None; + if is_receive { + let need = content_length + .map_or(t.cfg.push_reserve, |n| n.min(cap)) + .max(1); + let free = transport::free_bytes(&app.data_dir).unwrap_or(0); + match t.reserve.reserve(free, t.cfg.min_free, need) { + Some(h) => { + let h = Arc::new(Mutex::new(h)); + held = Some(h.clone()); + reservation = Some(Reservation { + held: h, + reserved: need, + data_dir: app.data_dir.clone(), + floor: t.cfg.min_free, + }); + } + None => return buffered(text(507, "Insufficient Storage\n")), + } + } + let launch_args = Launch { + app: app.clone(), + content_length: has_body.then_some(content_length).flatten(), + content_type: has_body + .then(|| transport::content_type(req.header("content-type"))) + .flatten(), + git_protocol: transport::protocol_header(req.header("git-protocol")), + remote: req.source, + plan, + permits, + held, + rt: tokio::runtime::Handle::current(), + }; + let started = match tokio::task::spawn_blocking(move || launch(launch_args)).await { + Ok(Ok(s)) => s, + Ok(Err(LaunchError::Gone)) => return buffered(transport::repo_not_found()), + Ok(Err(LaunchError::Stopped)) => { + return buffered(text(503, "Service Unavailable\n").with("retry-after", "5".into())) + } + Ok(Err(LaunchError::Io(e))) => { + eprintln!("sc-forge: could not start git: {e}"); + return buffered(text(500, "internal server error\n")); + } + Err(_) => return buffered(text(500, "internal server error\n")), + }; + let Started { + head, + body: out, + stdin, + cancel, + done, + } = started; + // From here, dropping this guard (the client went away, or we return an error) ends the child. + let guard = cancel.on_drop(); + let abort = Arc::new(AtomicU16::new(0)); + if !has_body { + drop(stdin); // nothing to send: the child sees EOF at once + } else { + tokio::spawn( + Pump { + body, + preload, + tx: stdin, + cancel: cancel.clone(), + done, + abort: abort.clone(), + idle: t.cfg.idle, + cap, + reservation, + } + .run(), + ); + } + let head = match head.await { + Ok(Ok(h)) => h, + other => { + let why = abort.load(Ordering::SeqCst); + let status = if why != 0 { why } else { 502 }; + if let Ok(Err(e)) = other { + eprintln!("sc-forge: git transport failed before answering: {e}"); + } + return buffered(text(status, "git transport failed\n")); + } + }; + // git marks loose objects and packs `public, max-age=31536000`; behind a shared cache that would + // serve an authorized object to anyone. Those and the RPC results get the private policy; git's + // own `no-cache, must-revalidate` on the dumb reads already forbids reuse without revalidation. + let public = head + .headers + .iter() + .any(|(k, v)| k == "cache-control" && v.to_ascii_lowercase().contains("public")); + let mut b = Response::builder().status(head.status); + for (k, v) in head.headers { + if public && k == "expires" { + continue; + } + // The RPC endpoints answer with Gitea's cache policy (recorded), not git's no-cache one. + let v = if k == "cache-control" && (is_rpc || public) { + transport::PRIVATE_CACHE.to_string() + } else { + v + }; + b = b.header(k, v); + } + b.body( + ChannelBody { + rx: out, + _guard: guard, + } + .boxed(), + ) + .unwrap_or_else(|_| fallback(502)) +} + +/// A plain (non-streamed) answer as a response, for the server's own use. +pub fn answer(r: crate::http::Resp) -> Response { + buffered(r) +} diff --git a/crates/forge/src/cli.rs b/crates/forge/src/cli.rs index d8372a92..6fac5b35 100644 --- a/crates/forge/src/cli.rs +++ b/crates/forge/src/cli.rs @@ -18,6 +18,11 @@ struct Cli { enum Cmd { /// Run the HTTP server. Serve(ServeArgs), + /// Internal: run a git hook (`pre-receive`); normally reached through the hooks symlink. + Hook { + #[arg(value_parser = ["pre-receive"])] + name: String, + }, /// Operate on the data directory directly (no network). Admin { #[command(subcommand)] @@ -56,6 +61,36 @@ struct ServeArgs { /// Concurrent `git` child processes (a busy server answers 503 rather than queueing). #[arg(long, default_value_t = 8)] git_concurrency: usize, + /// Git transports (clone, fetch, push) in flight at once, all repositories. + #[arg(long, default_value_t = 64)] + max_transports: usize, + /// Git transports in flight on one repository. + #[arg(long, default_value_t = 16)] + max_transports_per_repo: usize, + /// Seconds a git request waits for a transport slot before 503. + #[arg(long, default_value_t = 30)] + admission_wait: u64, + /// Wall-clock limit of one git transport, seconds. + #[arg(long, default_value_t = 1800)] + git_timeout: u64, + /// Seconds a git transport may wait on a stalled client. + #[arg(long, default_value_t = 120)] + git_idle_timeout: u64, + /// Largest push, bytes (also git's `receive.maxInputSize`). + #[arg(long, default_value_t = 2 << 30)] + max_pack_size: u64, + /// Largest fetch request, bytes, compressed and inflated. + #[arg(long, default_value_t = 16 << 20)] + max_request_bytes: u64, + /// Disk that must stay free beyond what pushes in flight have reserved, bytes. + #[arg(long, default_value_t = 1 << 30)] + min_free_bytes: u64, + /// Disk reserved up front for a push whose size is not announced, bytes. + #[arg(long, default_value_t = 256 << 20)] + push_reserve: u64, + /// Memory limit of each git transport child, bytes, Linux only (0 = none). + #[arg(long, default_value_t = 8 << 30)] + git_memory_limit: u64, /// Concurrent password verifications (Argon2 is memory-hard). #[arg(long, default_value_t = 4)] argon2_concurrency: usize, @@ -129,8 +164,18 @@ enum OrgCmd { /// Parse the command line and run `serve` or an `admin` command. pub fn run() -> Result<()> { + // git runs `/hooks/pre-receive`, a symlink to this binary: dispatch on argv[0] before any + // argument parsing, so the hook needs no shell and no quoting. + if std::env::args_os().next().is_some_and(|a| { + std::path::Path::new(&a) + .file_name() + .is_some_and(|n| n == "pre-receive") + }) { + std::process::exit(scl_forge::hook::run()); + } match Cli::parse().cmd { Cmd::Serve(a) => run_serve(a), + Cmd::Hook { .. } => std::process::exit(scl_forge::hook::run()), Cmd::Admin { cmd } => run_admin(cmd), } } @@ -208,6 +253,22 @@ fn run_serve(a: ServeArgs) -> Result<()> { if a.max_connections == 0 { bail!("--max-connections must be at least 1"); } + for (flag, v) in [ + ("--max-transports", a.max_transports as u64), + ( + "--max-transports-per-repo", + a.max_transports_per_repo as u64, + ), + ("--admission-wait", a.admission_wait), + ("--git-timeout", a.git_timeout), + ("--git-idle-timeout", a.git_idle_timeout), + ("--max-pack-size", a.max_pack_size), + ("--max-request-bytes", a.max_request_bytes), + ] { + if v == 0 { + bail!("{flag} must be at least 1"); + } + } bind_gate(&a.listen, a.behind_proxy, a.allow_plaintext).map_err(anyhow::Error::msg)?; let cidrs = a .trusted_proxies @@ -242,13 +303,33 @@ fn run_serve(a: ServeArgs) -> Result<()> { )?); let git_version = scl_forge::gitproc::check_version(&git).map_err(anyhow::Error::msg)?; eprintln!("sc-forge: {git_version}"); + scl_forge::hook::install( + &std::fs::canonicalize(&a.data_dir)?, + &std::env::current_exe().context("cannot find the sc-forge binary")?, + )?; let life = scl_forge::lifecycle::Lifecycle::new(&a.data_dir, git)?; // Resolve whatever a crash left, before the listener is bound. let report = life.reconcile(&db::connect(&a.data_dir)?, scl_forge::timefmt::now())?; if report != scl_forge::lifecycle::Report::default() { eprintln!("sc-forge: reconciled {report:?}"); } - let app = make_app(a.data_dir, &public_url, proxies, a.argon2_concurrency, life); + let tcfg = scl_forge::cgi::Cfg { + max_transports: a.max_transports, + per_repo: a.max_transports_per_repo, + admission_wait: Duration::from_secs(a.admission_wait), + timeout: Duration::from_secs(a.git_timeout), + idle: Duration::from_secs(a.git_idle_timeout), + max_pack: a.max_pack_size, + max_request: a.max_request_bytes, + min_free: a.min_free_bytes, + push_reserve: a.push_reserve, + git_memory: a.git_memory_limit, + ..Default::default() + }; + let mut app = make_app(a.data_dir, &public_url, proxies, a.argon2_concurrency, life); + std::sync::Arc::get_mut(&mut app) + .expect("the app has no other owner yet") + .transports = scl_forge::cgi::Transports::new(tcfg); let limits = Limits { max_body: a.max_body_bytes, max_connections: a.max_connections, diff --git a/crates/forge/src/db.rs b/crates/forge/src/db.rs index 4fe97336..0d56eff5 100644 --- a/crates/forge/src/db.rs +++ b/crates/forge/src/db.rs @@ -147,6 +147,25 @@ fn ensure_dir(data_dir: &Path) -> Result<(), DbError> { Ok(()) } +/// Open the existing database read-only, for the `pre-receive` hook: never creates the file, never +/// migrates, never takes the data-directory lock, and refuses unless the schema is exactly this +/// binary's (a binary replaced in place without a restart fails closed). +pub fn open_readonly(data_dir: &Path) -> Result { + let conn = Connection::open_with_flags( + db_path(data_dir), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?; + conn.busy_timeout(std::time::Duration::from_secs(5))?; + conn.pragma_update(None, "query_only", "ON")?; + let have = version(&conn)?; + if have != SCHEMA_VERSION { + return Err(DbError::Schema(format!( + "database schema {have} differs from this binary's {SCHEMA_VERSION}" + ))); + } + Ok(conn) +} + /// Open for `serve`: migrate forward, refuse a newer schema. pub fn open_for_serve(data_dir: &Path) -> Result<(), DbError> { ensure_dir(data_dir)?; @@ -199,6 +218,29 @@ mod tests { assert!(!d.exists()); } + #[test] + fn the_readonly_opener_never_creates_migrates_or_writes() { + let d = tmp("readonly"); + assert!( + open_readonly(&d).is_err(), + "no database: nothing is created" + ); + assert!(!db_path(&d).exists()); + drop(open_for_admin(&d).unwrap()); + let ro = open_readonly(&d).unwrap(); + assert!(ro.execute("INSERT INTO users (login, lower_login, kind, created_unix) VALUES ('x','x','user',1)", []).is_err(), "writes are refused"); + drop(ro); + let c = connect(&d).unwrap(); + c.pragma_update(None, "user_version", SCHEMA_VERSION + 1) + .unwrap(); + drop(c); + assert!( + matches!(open_readonly(&d), Err(DbError::Schema(_))), + "a schema that is not this binary's fails closed" + ); + let _ = std::fs::remove_dir_all(&d); + } + #[test] fn connection_settings_are_durable() { let d = tmp("pragmas"); diff --git a/crates/forge/src/gitproc.rs b/crates/forge/src/gitproc.rs index 4cc8cc61..e9981c02 100644 --- a/crates/forge/src/gitproc.rs +++ b/crates/forge/src/gitproc.rs @@ -139,6 +139,16 @@ impl Git { /// The environment of every spawn: built from scratch, nothing inherited. pub fn env(&self, identity: Option<&Identity>) -> Vec<(String, String)> { + self.env_with(identity, &[]) + } + + /// Like [`Git::env`], plus per-spawn git config entries (`key`, `value`) appended to the + /// pinned ones (the transport adds its upload/receive settings this way). + pub fn env_with( + &self, + identity: Option<&Identity>, + extra: &[(&str, String)], + ) -> Vec<(String, String)> { let d = self.data_dir.display().to_string(); let mut env: Vec<(String, String)> = vec![ ("PATH".into(), "/usr/bin:/bin".into()), @@ -151,17 +161,18 @@ impl Git { ("GIT_TERMINAL_PROMPT".into(), "0".into()), ("GIT_CEILING_DIRECTORIES".into(), d.clone()), ]; - let config = [ - ("core.hooksPath", format!("{d}/hooks")), - ("core.fsync", "all".to_string()), - ("core.fsyncMethod", "fsync".to_string()), - ("core.useReplaceRefs", "false".to_string()), - ("gc.auto", "0".to_string()), - ("receive.fsckObjects", "true".to_string()), + let mut config: Vec<(String, String)> = vec![ + ("core.hooksPath".into(), format!("{d}/hooks")), + ("core.fsync".into(), "all".into()), + ("core.fsyncMethod".into(), "fsync".into()), + ("core.useReplaceRefs".into(), "false".into()), + ("gc.auto".into(), "0".into()), + ("receive.fsckObjects".into(), "true".into()), ]; + config.extend(extra.iter().map(|(k, v)| (k.to_string(), v.clone()))); env.push(("GIT_CONFIG_COUNT".into(), config.len().to_string())); for (i, (k, v)) in config.into_iter().enumerate() { - env.push((format!("GIT_CONFIG_KEY_{i}"), k.into())); + env.push((format!("GIT_CONFIG_KEY_{i}"), k)); env.push((format!("GIT_CONFIG_VALUE_{i}"), v)); } if let Some(id) = identity { @@ -173,6 +184,11 @@ impl Git { env } + /// The git executable (absolute). + pub fn bin(&self) -> &Path { + &self.bin + } + /// Run git as described by `spec`. pub fn run(&self, spec: &Spec) -> Result { self.run_program(self.bin.as_os_str(), spec) @@ -290,7 +306,7 @@ impl Git { /// Has child `pid` exited? Uses `waitid(WEXITED | WNOHANG | WNOWAIT)`, which reports exit /// without reaping, so the pid stays reserved until the caller waits on the child. -fn exited_unreaped(pid: i32) -> std::io::Result { +pub(crate) fn exited_unreaped(pid: i32) -> std::io::Result { // SAFETY: an all-zero siginfo_t is a valid value for waitid to fill in. let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; // SAFETY: `info` is a valid out-pointer; the options only observe, never reap. diff --git a/crates/forge/src/hook.rs b/crates/forge/src/hook.rs new file mode 100644 index 00000000..29c3674f --- /dev/null +++ b/crates/forge/src/hook.rs @@ -0,0 +1,213 @@ +//! The `pre-receive` hook, run by `git receive-pack` as the symlink `/hooks/pre-receive` +//! (argv[0] `pre-receive`) or as `sc-forge hook pre-receive`. It reads ` ` lines on +//! stdin, asks [`crate::policy`] about each, and exits non-zero (refusing the whole push) with one +//! `remote:` line per refusal. It fails closed: a missing environment, an unreadable database, a +//! repository that is not `live` all refuse the push. +//! +//! Trust boundary: the hook runs with the server's user and read access to the database; its code +//! cannot be replaced by repository content (`core.hooksPath` is pinned per spawn and repositories +//! are created without a hooks directory), and it trusts nothing from git but the three stdin fields. + +use crate::{db, policy, repos}; +use std::io::Read; +use std::path::Path; + +/// Read git's update lines from `r` as raw bytes. Any read error or any line that is not valid UTF-8 +/// refuses the whole push: stopping at the first bad line would leave the rest unchecked (fail open). +pub fn read_updates(mut r: impl Read) -> Result, String> { + let mut raw = Vec::new(); + r.read_to_end(&mut raw) + .map_err(|e| format!("cannot read the update lines: {e}"))?; + raw.split(|b| *b == b'\n') + .filter(|l| !l.is_empty()) + .map(|l| { + String::from_utf8(l.to_vec()) + .map_err(|_| "an update line is not valid UTF-8".to_string()) + }) + .collect() +} + +/// Decide a whole push. `lines` are git's ` ` records; returns every refusal text. +pub fn decide(lines: &[String], default_branch: &str, live: bool) -> Vec { + if !live { + return vec!["refusing the push: the repository is not available".to_string()]; + } + let mut out = Vec::new(); + for l in lines { + let mut it = l.splitn(3, ' '); + match (it.next(), it.next(), it.next()) { + (Some(_old), Some(new), Some(name)) if !name.is_empty() => { + if let Err(e) = policy::check_update(name, new, default_branch) { + out.push(e); + } + } + _ => out.push(format!("refusing the push: malformed update line {l:?}")), + } + } + out +} + +/// Look up the repository the hook runs for: `(default_branch, live)`. +pub fn lookup(data_dir: &Path, repo_id: i64) -> Result<(String, bool), String> { + let conn = db::open_readonly(data_dir).map_err(|e| e.to_string())?; + match repos::find_by_id(&conn, repo_id).map_err(|e| e.to_string())? { + Some(r) => Ok((r.default_branch, r.state == "live")), + None => Err(format!("repository {repo_id} does not exist")), + } +} + +/// Point `/hooks/pre-receive` at `exe` (this binary): a symlink, so git runs the hook with no +/// shell and the path needs no quoting. Replaced atomically, at every start, because the binary's +/// path can change between runs. +pub fn install(data_dir: &Path, exe: &Path) -> std::io::Result<()> { + let hooks = data_dir.join("hooks"); + std::fs::create_dir_all(&hooks)?; + let tmp = hooks.join(".pre-receive.new"); + let _ = std::fs::remove_file(&tmp); + std::os::unix::fs::symlink(exe, &tmp)?; + std::fs::rename(&tmp, hooks.join("pre-receive")) +} + +/// Entry point: returns the process exit code. +pub fn run() -> i32 { + let (Some(data), Some(id)) = ( + std::env::var_os("FORGE_DATA_DIR"), + std::env::var("FORGE_REPO_ID") + .ok() + .and_then(|v| v.parse::().ok()), + ) else { + eprintln!( + "sc-forge pre-receive: refusing the push: missing FORGE_DATA_DIR or FORGE_REPO_ID" + ); + return 1; + }; + let lines = match read_updates(std::io::stdin().lock()) { + Ok(l) => l, + Err(e) => { + eprintln!("sc-forge pre-receive: refusing the push: {e}"); + return 1; + } + }; + let (default_branch, live) = match lookup(Path::new(&data), id) { + Ok(x) => x, + Err(e) => { + eprintln!("sc-forge pre-receive: refusing the push: {e}"); + return 1; + } + }; + let refusals = decide(&lines, &default_branch, live); + for r in &refusals { + eprintln!("{r}"); + } + // Tell the server that this push was let through, so it stamps the repository only for a push + // that could change a ref (git exits 0 after reporting a rejection, too). + if refusals.is_empty() && !lines.is_empty() { + if let Some(m) = std::env::var_os("FORGE_ACCEPT_MARKER") { + let _ = std::fs::write(m, b""); + } + } + i32::from(!refusals.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const Z: &str = "0000000000000000000000000000000000000000"; + const A: &str = "1111111111111111111111111111111111111111"; + + fn l(new: &str, r: &str) -> String { + format!("{A} {new} {r}") + } + + #[test] + fn a_clean_push_is_accepted() { + assert!(decide( + &[l(A, "refs/heads/feat"), l(A, "refs/tags/v1")], + "main", + true + ) + .is_empty()); + } + + #[test] + fn one_bad_ref_refuses_the_whole_push_and_every_refusal_is_reported() { + let out = decide( + &[ + l(A, "refs/heads/ok"), + l(A, "refs/replace/x"), + l(Z, "refs/heads/MAIN"), + l(A, "refs/pull/1/head"), + ], + "main", + true, + ); + assert_eq!(out.len(), 3, "{out:?}"); + assert!( + out[0].contains("refs/replace/x") + && out[1].contains("default branch") + && out[2].contains("refs/pull/1/head") + ); + } + + #[test] + fn a_repository_that_is_not_live_refuses_everything() { + assert_eq!(decide(&[l(A, "refs/heads/x")], "main", false).len(), 1); + } + + #[test] + fn malformed_lines_fail_closed() { + for bad in ["", "onlyone", "a b", "a b "] { + assert_eq!(decide(&[bad.to_string()], "main", true).len(), 1, "{bad:?}"); + } + } + + #[test] + fn unreadable_or_non_utf8_update_lines_refuse_the_whole_push() { + assert_eq!( + read_updates(&b"a b refs/heads/x\nc d refs/tags/y\n"[..]) + .unwrap() + .len(), + 2 + ); + assert!( + read_updates(&b"a b refs/heads/ok\na b refs/replace/\xff\n"[..]).is_err(), + "a bad line must not hide the lines after it" + ); + assert!(read_updates(&b"\xff\na b refs/replace/x\n"[..]).is_err()); + struct Broken; + impl Read for Broken { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("boom")) + } + } + assert!(read_updates(Broken).is_err()); + } + + #[test] + fn the_lookup_reports_live_state_and_the_default_branch() { + let d = std::env::temp_dir().join(format!("scl-forge-hook-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + let conn = db::open_for_admin(&d).unwrap(); + let owner = crate::store::create_account( + &conn, + &crate::store::NewAccount { + login: "o".into(), + visibility: "public".into(), + ..Default::default() + }, + 1, + ) + .unwrap(); + let r = repos::insert_creating(&conn, owner.id, "r", "trunk", "", "", 1).unwrap(); + assert_eq!( + lookup(&d, r.id).unwrap(), + ("trunk".to_string(), false), + "creating is not live" + ); + repos::set_state(&conn, r.id, "creating", "live", 2).unwrap(); + assert_eq!(lookup(&d, r.id).unwrap(), ("trunk".to_string(), true)); + assert!(lookup(&d, 999).is_err()); + let _ = std::fs::remove_dir_all(&d); + } +} diff --git a/crates/forge/src/lib.rs b/crates/forge/src/lib.rs index a74186a6..7475b480 100644 --- a/crates/forge/src/lib.rs +++ b/crates/forge/src/lib.rs @@ -10,14 +10,18 @@ pub mod admin; pub mod api; pub mod auth; +pub mod cgi; pub mod db; pub mod gitproc; pub mod gitread; +pub mod hook; pub mod http; pub mod lifecycle; pub mod limits; pub mod names; +pub mod policy; pub mod refname; +pub mod registry; pub mod repo_api; pub mod repos; pub mod routes; @@ -25,3 +29,4 @@ pub mod scope; pub mod server; pub mod store; pub mod timefmt; +pub mod transport; diff --git a/crates/forge/src/lifecycle.rs b/crates/forge/src/lifecycle.rs index 55cff1e8..efa5041f 100644 --- a/crates/forge/src/lifecycle.rs +++ b/crates/forge/src/lifecycle.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; use std::fs::File; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; /// Exclusive hold on the data directory for the life of the process. pub struct DirLock(#[allow(dead_code)] File); @@ -88,6 +89,8 @@ pub struct Report { pub struct Lifecycle { data_dir: PathBuf, pub git: Arc, + /// Live git transports by repository, cancelled by `delete` and at shutdown. + pub registry: Arc, locks: Mutex>>>, #[cfg(test)] crash_at: Mutex>, @@ -108,6 +111,7 @@ impl Lifecycle { Ok(Lifecycle { data_dir, git, + registry: Arc::default(), locks: Mutex::default(), #[cfg(test)] crash_at: Mutex::default(), @@ -326,9 +330,18 @@ impl Lifecycle { return Err(LifecycleError::Gone); } self.step("marked")?; + // The row says `deleting`, so no new transport registers; end the ones running and wait for + // them to let go before the directory moves. One that will not end keeps the repository. + self.registry.close(repo.id); + if !self.registry.wait_idle(repo.id, Duration::from_secs(10)) { + self.registry.reopen(repo.id); + let _ = repos::set_state(conn, repo.id, "deleting", "live", now); + return Err(LifecycleError::Git(GitError::Busy)); + } let (from, to) = (self.repo_dir(repo.id), self.trash_dir(repo.id)); if let Err(e) = std::fs::rename(&from, &to) { // Nothing moved: put the row back so the repository is usable again. + self.registry.reopen(repo.id); let _ = repos::set_state(conn, repo.id, "deleting", "live", now); return Err(e.into()); } @@ -343,6 +356,7 @@ impl Lifecycle { repo.id ); } + self.registry.forget(repo.id); self.locks .lock() .unwrap_or_else(|e| e.into_inner()) @@ -414,7 +428,7 @@ impl Lifecycle { .filter(|r| r.state == "live") .ok_or(LifecycleError::Gone)?; // Case-insensitive: on a case-insensitive filesystem `MAIN` aliases the loose ref `main`. - if current.default_branch.eq_ignore_ascii_case(branch) { + if crate::policy::same_branch_name(¤t.default_branch, branch) { return Err(LifecycleError::GitFailed( "refusing to delete the default branch".into(), )); diff --git a/crates/forge/src/policy.rs b/crates/forge/src/policy.rs new file mode 100644 index 00000000..890690bf --- /dev/null +++ b/crates/forge/src/policy.rs @@ -0,0 +1,125 @@ +//! The ref-update policy every mutation path shares. F2b enforces it in the `pre-receive` hook; +//! F3's contents writes and F4's merges call the same function, and branch protection rules (F4) +//! extend it here, so there is one place that decides whether a ref may move. + +/// Why an update is refused (the text git shows to the pusher as `remote: `). +pub type Refusal = String; + +/// Is `new` the all-zero id (a deletion)? Any length: sha1 and sha256 repositories alike. +pub fn is_delete(new: &str) -> bool { + !new.is_empty() && new.bytes().all(|b| b == b'0') +} + +/// Do two branch names name the same branch on a case-insensitive filesystem? Compared by Unicode +/// lowercase (there, `MAIN` and `Ünï` alias their lowercase forms). +/// shortcut: Unicode normalization (NFC vs NFD) aliasing on APFS/HFS+ is not handled; production is +/// Linux (case- and normalization-sensitive), macOS is development only. +pub fn same_branch_name(a: &str, b: &str) -> bool { + a.to_lowercase() == b.to_lowercase() +} + +/// May a client push `name` (`old` -> `new`) to a repository whose default branch is +/// `default_branch`? Only `refs/heads/*` and `refs/tags/*` are writable; `refs/replace/*`, +/// `refs/forge/*`, `refs/pull/*`, `refs/notes/*` and anything else is refused. The default branch +/// cannot be deleted, compared case-insensitively (on a case-insensitive filesystem `MAIN` aliases +/// the loose ref `main`). +pub fn check_update(name: &str, new: &str, default_branch: &str) -> Result<(), Refusal> { + if let Some(branch) = name.strip_prefix("refs/heads/") { + if is_delete(new) && same_branch_name(branch, default_branch) { + return Err(format!("refusing to delete the default branch {branch}")); + } + return Ok(()); + } + if name.starts_with("refs/tags/") { + return Ok(()); + } + Err(format!( + "refusing to update {name}: only refs/heads/* and refs/tags/* are writable" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + const Z: &str = "0000000000000000000000000000000000000000"; + const A: &str = "1111111111111111111111111111111111111111"; + + #[test] + fn branches_and_tags_are_writable() { + for r in [ + "refs/heads/main", + "refs/heads/feat/x", + "refs/tags/v1", + "refs/heads/MAIN-2", + ] { + assert!(check_update(r, A, "main").is_ok(), "{r}"); + assert!( + check_update(r, Z, "dev").is_ok() || r.ends_with("dev"), + "{r}: deleting a non-default is allowed" + ); + } + } + + #[test] + fn every_other_namespace_is_refused_with_the_recorded_text() { + for r in [ + "refs/replace/1111111111111111111111111111111111111111", + "refs/forge/pull/1/head", + "refs/pull/1/head", + "refs/notes/commits", + "refs/changes/01/1/1", + "refs/for/main", + "refs/keep-around/abc", + "HEAD", + "refs/heads", + "refs/headsx/main", + "refs/tag/v1", + "FETCH_HEAD", + "", + ] { + let e = check_update(r, A, "main").unwrap_err(); + assert_eq!( + e, + format!("refusing to update {r}: only refs/heads/* and refs/tags/* are writable"), + "{r:?}" + ); + } + // a name that merely contains `..` after the allowed prefix is git's to reject (check-ref-format); the policy + // only decides the namespace, and `refs/heads/x/../../replace/y` is not under another prefix as a ref name + assert!( + check_update("refs/heads/../replace/y", A, "main").is_ok(), + "namespace check is a prefix check; git refuses the name itself" + ); + } + + #[test] + fn the_default_branch_cannot_be_deleted_even_by_a_case_alias() { + for r in ["refs/heads/main", "refs/heads/MAIN", "refs/heads/Main"] { + assert!(check_update(r, Z, "main").is_err(), "{r}"); + } + assert!( + check_update("refs/heads/main", A, "main").is_ok(), + "updating it is fine" + ); + assert!( + check_update("refs/heads/main", "0".repeat(64).as_str(), "main").is_err(), + "sha256 all-zero id is a delete too" + ); + assert!(check_update("refs/heads/mainx", Z, "main").is_ok()); + assert!( + check_update("refs/tags/main", Z, "main").is_ok(), + "a tag named like the default branch is not the branch" + ); + } + + #[test] + fn deletion_means_an_all_zero_id_of_any_length() { + assert!(is_delete(Z) && is_delete("0") && is_delete(&"0".repeat(64))); + assert!( + !is_delete("") + && !is_delete(A) + && !is_delete("0000000000000000000000000000000000000001") + ); + } +} diff --git a/crates/forge/src/registry.rs b/crates/forge/src/registry.rs new file mode 100644 index 00000000..26a9ce37 --- /dev/null +++ b/crates/forge/src/registry.rs @@ -0,0 +1,288 @@ +//! Who is running against which repository, and the cancel handle that ends them. +//! +//! The registry holds a [`Cancel`] per live transport, never a process-group id: only the +//! supervisor thread that owns the child signals its group (while the leader is still an unreaped +//! zombie, so the id cannot have been reused), and it does so when it sees the flag. `delete` +//! closes a repository's slot, cancels what is in it and waits for the transports to unregister +//! before it renames the directory; a transport that arrives after the close is refused. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +/// A cancellation request shared between the registry, the supervisor and the pumps. Setting it +/// is idempotent; the async side awaits [`Cancel::cancelled`], threads poll [`Cancel::is_set`]. +#[derive(Clone, Default)] +pub struct Cancel(Arc); + +#[derive(Default)] +struct CancelInner { + flag: AtomicBool, + notify: tokio::sync::Notify, +} + +impl Cancel { + pub fn cancel(&self) { + self.0.flag.store(true, Ordering::SeqCst); + self.0.notify.notify_waiters(); + } + + pub fn is_set(&self) -> bool { + self.0.flag.load(Ordering::SeqCst) + } + + /// Resolves once cancelled (immediately if it already was). + pub async fn cancelled(&self) { + // The `Notified` future is created before the flag is read, so a `cancel` between the two + // is not lost. + let n = self.0.notify.notified(); + tokio::pin!(n); + n.as_mut().enable(); + if self.is_set() { + return; + } + n.await; + } + + /// Cancels when dropped: held by whatever owns the client side of a transport. + pub fn on_drop(&self) -> CancelOnDrop { + CancelOnDrop(self.clone()) + } +} + +pub struct CancelOnDrop(Cancel); + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel(); + } +} + +#[derive(Default)] +struct Slot { + closed: bool, + live: HashMap, +} + +/// Live transports by repository id. +#[derive(Default)] +pub struct Registry { + slots: Mutex>, + /// Set at shutdown: nothing registers afterwards. + stopped: AtomicBool, + idle: Condvar, + next: AtomicU64, +} + +/// A registered transport; unregisters on drop. +pub struct Registration { + reg: Arc, + repo: i64, + key: u64, + pub cancel: Cancel, +} + +impl Registry { + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.slots.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Register a transport on `repo`. `still_live` runs under the registry lock, after the closed + /// check, so a repository that `delete` has marked cannot slip a transport in between. `None` + /// when the slot is closed or the repository is not live. + pub fn register( + self: &Arc, + repo: i64, + still_live: impl FnOnce() -> bool, + ) -> Option { + let mut slots = self.lock(); + if self.stopped.load(Ordering::SeqCst) + || slots.get(&repo).is_some_and(|s| s.closed) + || !still_live() + { + return None; + } + let key = self.next.fetch_add(1, Ordering::SeqCst); + let cancel = Cancel::default(); + slots + .entry(repo) + .or_default() + .live + .insert(key, cancel.clone()); + Some(Registration { + reg: self.clone(), + repo, + key, + cancel, + }) + } + + /// Refuse new transports on `repo` and cancel the live ones. + pub fn close(&self, repo: i64) { + let mut slots = self.lock(); + let slot = slots.entry(repo).or_default(); + slot.closed = true; + slot.live.values().for_each(Cancel::cancel); + } + + /// Allow transports on `repo` again (a delete that did not go through). + pub fn reopen(&self, repo: i64) { + let mut slots = self.lock(); + if let Some(s) = slots.get_mut(&repo) { + s.closed = false; + if s.live.is_empty() { + slots.remove(&repo); + } + } + } + + /// Wait up to `limit` for every transport on `repo` to unregister; `false` if some survive. + pub fn wait_idle(&self, repo: i64, limit: Duration) -> bool { + let end = Instant::now() + limit; + let mut slots = self.lock(); + while slots.get(&repo).is_some_and(|s| !s.live.is_empty()) { + let Some(left) = end.checked_duration_since(Instant::now()) else { + return false; + }; + slots = self + .idle + .wait_timeout(slots, left) + .unwrap_or_else(|e| e.into_inner()) + .0; + } + true + } + + /// Drop the (closed) slot of a repository that no longer exists. + pub fn forget(&self, repo: i64) { + self.lock().remove(&repo); + } + + /// Has shutdown begun? + pub fn is_stopped(&self) -> bool { + self.stopped.load(Ordering::SeqCst) + } + + /// Cancel every live transport and refuse new ones (shutdown): a request still queued for + /// admission must not start a child after the sweep. + pub fn cancel_all(&self) { + let slots = self.lock(); + self.stopped.store(true, Ordering::SeqCst); + slots + .values() + .flat_map(|s| s.live.values()) + .for_each(Cancel::cancel); + } + + /// Wait up to `limit` until no transport at all is registered. + pub fn wait_all_idle(&self, limit: Duration) -> bool { + let end = Instant::now() + limit; + let mut slots = self.lock(); + while slots.values().any(|s| !s.live.is_empty()) { + let Some(left) = end.checked_duration_since(Instant::now()) else { + return false; + }; + slots = self + .idle + .wait_timeout(slots, left) + .unwrap_or_else(|e| e.into_inner()) + .0; + } + true + } + + /// How many transports are registered (tests, `/healthz`-style probes). + pub fn live(&self) -> usize { + self.lock().values().map(|s| s.live.len()).sum() + } +} + +impl Drop for Registration { + fn drop(&mut self) { + let mut slots = self.reg.lock(); + if let Some(s) = slots.get_mut(&self.repo) { + s.live.remove(&self.key); + if s.live.is_empty() && !s.closed { + slots.remove(&self.repo); + } + } + self.reg.idle.notify_all(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_closed_slot_refuses_and_cancels() { + let r = Arc::new(Registry::default()); + let a = r.register(1, || true).expect("open"); + assert!(!a.cancel.is_set()); + r.close(1); + assert!(a.cancel.is_set(), "close cancels what is live"); + assert!(r.register(1, || true).is_none(), "closed refuses"); + assert!( + r.register(2, || true).is_some(), + "other repositories are unaffected" + ); + assert!( + !r.wait_idle(1, Duration::from_millis(20)), + "a live transport keeps it busy" + ); + drop(a); + assert!(r.wait_idle(1, Duration::from_millis(20))); + r.reopen(1); + assert!(r.register(1, || true).is_some()); + } + + #[test] + fn the_liveness_check_runs_under_the_lock_and_can_refuse() { + let r = Arc::new(Registry::default()); + assert!(r.register(1, || false).is_none()); + assert_eq!(r.live(), 0); + } + + #[test] + fn wait_idle_wakes_when_the_last_transport_leaves() { + let r = Arc::new(Registry::default()); + let a = r.register(7, || true).unwrap(); + r.close(7); + let t = std::thread::spawn({ + let r = r.clone(); + move || r.wait_idle(7, Duration::from_secs(5)) + }); + std::thread::sleep(Duration::from_millis(30)); + drop(a); + assert!(t.join().unwrap()); + } + + #[test] + fn shutdown_cancels_everything() { + let r = Arc::new(Registry::default()); + let a = r.register(1, || true).unwrap(); + let b = r.register(2, || true).unwrap(); + r.cancel_all(); + assert!(a.cancel.is_set() && b.cancel.is_set()); + assert!(!r.wait_all_idle(Duration::from_millis(10))); + drop((a, b)); + assert!(r.wait_all_idle(Duration::from_millis(10))); + } + + #[tokio::test] + async fn cancelled_resolves_after_and_before_the_cancel() { + let c = Cancel::default(); + let waiter = tokio::spawn({ + let c = c.clone(); + async move { c.cancelled().await } + }); + tokio::time::sleep(Duration::from_millis(10)).await; + c.cancel(); + waiter.await.unwrap(); + c.cancelled().await; // already set: returns at once + let d = Cancel::default(); + drop(d.on_drop()); + assert!(d.is_set()); + } +} diff --git a/crates/forge/src/repo_api.rs b/crates/forge/src/repo_api.rs index d1d67fb3..f9d18584 100644 --- a/crates/forge/src/repo_api.rs +++ b/crates/forge/src/repo_api.rs @@ -618,7 +618,11 @@ fn branch_json(ctx: &Ctx, a: &Access, br: &gitread::Branch) -> Result| json!({"name": p.name, "email": p.email, "username": login.unwrap_or_default()}); let full = format!("{}/{}", a.repo.owner, a.repo.name); Ok(json!({ "name": br.name, @@ -626,8 +630,8 @@ fn branch_json(ctx: &Ctx, a: &Access, br: &gitread::Branch) -> Result Resp { if !refname::valid_branch(&name) { return no_branch(ctx, &name); } - if name.eq_ignore_ascii_case(&a.repo.default_branch) { + if crate::policy::same_branch_name(&name, &a.repo.default_branch) { return error(&ctx.app.base, 403, "can not delete default branch"); } match ctx.app.life.delete_branch(ctx.conn, &a.repo, &name) { diff --git a/crates/forge/src/repos.rs b/crates/forge/src/repos.rs index be5dcd50..a1def8f9 100644 --- a/crates/forge/src/repos.rs +++ b/crates/forge/src/repos.rs @@ -139,6 +139,15 @@ pub fn set_state( )? > 0) } +/// Stamp `updated_unix` after a push (the server does it itself; no post-receive hook exists). +pub fn touch(conn: &Connection, id: i64, now: i64) -> rusqlite::Result<()> { + conn.execute( + "UPDATE repos SET updated_unix = ?2 WHERE id = ?1 AND state = 'live'", + params![id, now], + )?; + Ok(()) +} + /// Delete the row (collaborators cascade). pub fn delete_row(conn: &Connection, id: i64) -> rusqlite::Result<()> { conn.execute("DELETE FROM repos WHERE id = ?1", [id])?; diff --git a/crates/forge/src/routes.rs b/crates/forge/src/routes.rs index 75a4eba2..64a13b8a 100644 --- a/crates/forge/src/routes.rs +++ b/crates/forge/src/routes.rs @@ -23,6 +23,8 @@ pub struct App { pub proxies: Proxies, /// Repository directories, the git runner and per-repository serialization. pub life: Lifecycle, + /// Admission and limits of the git transport. + pub transports: crate::cgi::Transports, } impl App { @@ -42,6 +44,7 @@ impl App { gate: Gate::new(argon_slots), proxies, life, + transports: crate::cgi::Transports::new(crate::cgi::Cfg::default()), } } diff --git a/crates/forge/src/server.rs b/crates/forge/src/server.rs index 6dd1cf72..32748fd8 100644 --- a/crates/forge/src/server.rs +++ b/crates/forge/src/server.rs @@ -2,9 +2,11 @@ //! timeout and connection cap here; routing, SQLite and Argon2 run on blocking //! threads in `routes::dispatch`. +use crate::cgi::{self, RespBody}; use crate::http::{Req, Resp}; use crate::limits::Proxies; use crate::routes::{dispatch, App}; +use crate::transport; use http_body_util::{BodyExt, Full, Limited}; use hyper::body::{Bytes, Incoming}; use hyper::service::service_fn; @@ -63,13 +65,35 @@ async fn handle( limits: Limits, peer: IpAddr, req: Request, -) -> Result>, std::convert::Infallible> { +) -> Result, std::convert::Infallible> { let (parts, body) = req.into_parts(); let headers = parts .headers .iter() .filter_map(|(k, v)| Some((k.as_str().to_string(), v.to_str().ok()?.to_string()))) .collect::>(); + // Git transport paths stream their body: route them before the bounded read below. + match transport::parse(parts.uri.path()) { + transport::Route::NotGit => {} + transport::Route::Unknown => return Ok(cgi::answer(transport::not_found())), + transport::Route::Git(g) => { + let source = app.proxies.resolve( + peer, + headers + .get(app.proxies.header.as_deref().unwrap_or("")) + .map(String::as_str), + ); + let r = Req { + method: parts.method.as_str().to_string(), + path: parts.uri.path().to_string(), + query: parts.uri.query().unwrap_or("").to_string(), + headers, + body: Vec::new(), + source, + }; + return Ok(cgi::handle(app, r, body, g).await); + } + } let body = match tokio::time::timeout( limits.body_timeout, Limited::new(body, limits.max_body).collect(), @@ -120,7 +144,7 @@ async fn handle( Ok(to_response(resp)) } -fn to_response(r: Resp) -> Response> { +fn to_response(r: Resp) -> Response { let mut b = Response::builder() .status(r.status) .header("server", SERVER); @@ -128,8 +152,8 @@ fn to_response(r: Resp) -> Response> { b = b.header(k, v); } // A header value hyper refuses must not panic the connection task: answer 500. - b.body(Full::new(Bytes::from(r.body))).unwrap_or_else(|_| { - let mut e = Response::new(Full::new(Bytes::from_static( + b.body(full(Bytes::from(r.body))).unwrap_or_else(|_| { + let mut e = Response::new(full(Bytes::from_static( b"{\"message\":\"internal server error\"}", ))); *e.status_mut() = hyper::StatusCode::INTERNAL_SERVER_ERROR; @@ -137,6 +161,10 @@ fn to_response(r: Resp) -> Response> { }) } +fn full(b: Bytes) -> RespBody { + Full::new(b).map_err(|e| match e {}).boxed() +} + /// Parse and normalize `--public-url`: `http(s)://host[:port][/path]` with a real host, /// no user-info, no query or fragment. Returns the normalized base (percent-encoded, /// no trailing slash) that every URL in a body or `Link` header is built from, so a @@ -223,6 +251,11 @@ pub async fn serve( } } let _ = tokio::time::timeout(drain, async { while conns.join_next().await.is_some() {} }).await; + // Whatever git is still running at the deadline ends now; wait for the children to be reaped. + let registry = app.life.registry.clone(); + registry.cancel_all(); + let _ = + tokio::task::spawn_blocking(move || registry.wait_all_idle(Duration::from_secs(10))).await; } /// Build the shared state from CLI inputs. diff --git a/crates/forge/src/store.rs b/crates/forge/src/store.rs index aabcba32..f6ee13f3 100644 --- a/crates/forge/src/store.rs +++ b/crates/forge/src/store.rs @@ -142,6 +142,25 @@ pub fn list_users(conn: &Connection, page: i64, limit: i64) -> rusqlite::Result< Ok((rows.collect::>()?, total)) } +/// The login of the one user whose email is `email` (case-insensitive), as Gitea resolves a commit +/// author to an account. `None` for an empty address, no match, or an address shared by several +/// accounts (ambiguous: attribute to nobody). +pub fn login_by_email(conn: &Connection, email: &str) -> rusqlite::Result> { + if email.is_empty() { + return Ok(None); + } + let mut st = conn.prepare( + "SELECT login FROM users WHERE kind = 'user' AND ulower(email) = ulower(?1) LIMIT 2", + )?; + let found = st + .query_map([email], |r| r.get::<_, String>(0))? + .collect::, _>>()?; + Ok(match found.as_slice() { + [one] => Some(one.clone()), + _ => None, + }) +} + /// Partial update of a user; `None` leaves a field alone. #[derive(Default)] pub struct UserPatch { diff --git a/crates/forge/src/transport.rs b/crates/forge/src/transport.rs new file mode 100644 index 00000000..a2b3e794 --- /dev/null +++ b/crates/forge/src/transport.rs @@ -0,0 +1,735 @@ +//! Smart-HTTP git transport: `/{owner}/{repo}.git/...` served by `git http-backend` as a CGI. +//! +//! This module holds the parts that decide *what may be asked of git* and *how git's answer is +//! read*, as plain functions with tests: the path whitelist, the git-specific credential and +//! answer rules, the CGI header block parser, the environment filters, and the disk reservation +//! arithmetic. The process supervision lives in `cgi.rs`. +//! +//! The sub-path handed to the CGI is rebuilt from the whitelist, never from the client's string. + +use crate::auth::{self, Cred}; +use crate::http::{Req, Resp}; +use crate::repos::{self, Access, Perm}; +use crate::routes::App; +use crate::scope::{self, Cat, Level, Need}; +use crate::store::{self, User}; + +/// What a git request asks for. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Sub { + InfoRefs, + UploadPack, + ReceivePack, + Head, + InfoPacks, + /// `objects/xx/`: the relative path (validated hex). + Loose(String), + /// `objects/pack/pack-.pack|idx`. + Pack(String), +} + +impl Sub { + /// The `PATH_INFO` for the CGI, rebuilt from the whitelist. + pub fn path_info(&self) -> String { + match self { + Sub::InfoRefs => "/info/refs".into(), + Sub::UploadPack => "/git-upload-pack".into(), + Sub::ReceivePack => "/git-receive-pack".into(), + Sub::Head => "/HEAD".into(), + Sub::InfoPacks => "/objects/info/packs".into(), + Sub::Loose(p) | Sub::Pack(p) => format!("/{p}"), + } + } + + /// The HTTP method this sub-path accepts. + pub fn method(&self) -> &'static str { + match self { + Sub::UploadPack | Sub::ReceivePack => "POST", + _ => "GET", + } + } +} + +/// A parsed `/{owner}/{repo}.git/{sub}` request path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitPath { + pub owner: String, + pub repo: String, + pub sub: Sub, +} + +/// Is `p` a path this server routes to the transport? +#[derive(Debug, PartialEq, Eq)] +pub enum Route { + /// Not a git path at all (the ordinary router handles it). + NotGit, + /// A `/{o}/{r}.git/...` path that is not on the whitelist: 404. + Unknown, + Git(GitPath), +} + +fn name_chars_ok(s: &str) -> bool { + !s.is_empty() + && s.len() <= 255 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_')) +} + +fn is_hex(s: &str, len: &[usize]) -> bool { + len.contains(&s.len()) + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +fn sub_of(rest: &str) -> Option { + Some(match rest { + "info/refs" => Sub::InfoRefs, + "git-upload-pack" => Sub::UploadPack, + "git-receive-pack" => Sub::ReceivePack, + "HEAD" => Sub::Head, + "objects/info/packs" => Sub::InfoPacks, + _ => { + let parts: Vec<&str> = rest.split('/').collect(); + match parts.as_slice() { + ["objects", d, f] if is_hex(d, &[2]) && is_hex(f, &[38, 62]) => { + Sub::Loose(rest.to_string()) + } + ["objects", "pack", f] => { + let stem = f.strip_prefix("pack-")?; + let (hex, ext) = stem.rsplit_once('.')?; + (is_hex(hex, &[40, 64]) && matches!(ext, "pack" | "idx")) + .then(|| Sub::Pack(rest.to_string()))? + } + _ => return None, + } + } + }) +} + +/// Classify a request path. The owner and repository segments are percent-decoded and must be +/// plain name characters (so no `/`, `..` or control character can come out of a decode); the +/// rest must match the whitelist exactly, byte for byte, with no escapes. +pub fn parse(path: &str) -> Route { + let Some(p) = path.strip_prefix('/') else { + return Route::NotGit; + }; + let mut it = p.splitn(3, '/'); + let (Some(owner), Some(repo_git)) = (it.next(), it.next()) else { + return Route::NotGit; + }; + let Some(repo_enc) = repo_git.strip_suffix(".git") else { + return Route::NotGit; + }; + let rest = it.next().unwrap_or(""); + let (Some(owner), Some(repo)) = ( + crate::routes::percent_decode(owner), + crate::routes::percent_decode(repo_enc), + ) else { + return Route::Unknown; + }; + if !name_chars_ok(&owner) + || !name_chars_ok(&repo) + || [&owner, &repo].iter().any(|n| *n == "." || *n == "..") + { + return Route::Unknown; + } + match sub_of(rest) { + Some(sub) => Route::Git(GitPath { owner, repo, sub }), + None => Route::Unknown, + } +} + +/// The service an `info/refs` or RPC request names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Service { + Upload, + Receive, +} + +impl Service { + pub fn name(self) -> &'static str { + match self { + Service::Upload => "git-upload-pack", + Service::Receive => "git-receive-pack", + } + } +} + +/// An `info/refs` request without a valid smart `service` parameter. +#[derive(Debug, PartialEq, Eq)] +pub struct NotSmart; + +/// The service a path asks for: from the sub-path, or for `info/refs` from `?service=`. A smart +/// `info/refs` without a valid `service` (git's dumb listing) is not served: it would ignore +/// `transfer.hideRefs`. +pub fn service_of(sub: &Sub, query_service: Option<&str>) -> Result, NotSmart> { + match sub { + Sub::UploadPack => Ok(Some(Service::Upload)), + Sub::ReceivePack => Ok(Some(Service::Receive)), + Sub::InfoRefs => match query_service { + Some("git-upload-pack") => Ok(Some(Service::Upload)), + Some("git-receive-pack") => Ok(Some(Service::Receive)), + _ => Err(NotSmart), + }, + _ => Ok(None), + } +} + +// ---- answers (plain text, the recorded Gitea bodies) ---- + +const TEXT: &str = "text/plain;charset=utf-8"; + +/// The `Cache-Control` Gitea sends on its 401s and on the results of the RPC endpoints (recorded +/// V#1/3/16/18/20/21). +pub const PRIVATE_CACHE: &str = "max-age=0, private, must-revalidate, no-transform"; + +fn text(status: u16, body: &str) -> Resp { + Resp { + status, + headers: vec![("content-type", TEXT.into())], + body: body.as_bytes().to_vec(), + delay: std::time::Duration::ZERO, + } +} + +/// 401 for a request with no credentials: asks the client to authenticate. +pub fn unauthorized() -> Resp { + text(401, "Unauthorized\n") + .with("www-authenticate", "Basic realm=\"Gitea\"".into()) + .with("cache-control", PRIVATE_CACHE.into()) +} + +/// 401 for a credential that does not verify (unknown, expired, or a password). +pub fn bad_credentials() -> Resp { + text(401, "Verify\n").with("cache-control", PRIVATE_CACHE.into()) +} + +/// 403: authenticated, but the token's scope or the caller's permission is not enough. +pub fn forbidden() -> Resp { + text(403, "Forbidden\n") +} + +/// 404: no such repository, or none the caller may see (one answer). +pub fn repo_not_found() -> Resp { + text(404, "Repository not found") +} + +/// 404 for a path under a repository that is not on the whitelist. +pub fn not_found() -> Resp { + text(404, "Not Found\n") +} + +/// 405 for a method the sub-path does not accept. +pub fn method_not_allowed(allow: &str) -> Resp { + text(405, "Method Not Allowed\n").with("allow", allow.into()) +} + +// ---- credentials ---- + +/// The token a git request presents, in any recorded form (`token`/`Bearer` header, Basic with +/// the token as the password under any username, or the token as the username with an empty +/// password). A Basic password that is not a token is simply a token that will not verify: git +/// transport never accepts a user password. +pub fn credential(cred: &Cred) -> Option<&str> { + match cred { + Cred::Anonymous => None, + Cred::Header(t) => Some(t), + Cred::Basic { user, pass } => Some(if pass.is_empty() { user } else { pass }), + } +} + +/// What a request is allowed to do, decided before any process exists. +pub struct Plan { + pub user: User, + pub access: Access, + pub path: GitPath, + pub service: Option, + /// `REMOTE_USER` may be set (the write check passed): receive-pack only. + pub may_write: bool, +} + +/// Authenticate, check scope, resolve the repository and the caller's permission, in the one order +/// every git request follows: credentials (401), method (405), a valid smart service (404), scope +/// (403), repository and existence (404 for missing, not-live or invisible alike), permission (403). +pub fn authorize(app: &App, req: &Req, path: GitPath) -> Result { + let conn = crate::db::connect(&app.data_dir).map_err(|e| internal(app, &e))?; + let cred = auth::parse(req); + let Some(secret) = credential(&cred) else { + return Err(unauthorized()); + }; + let (user, token) = match store::resolve_token(&conn, secret, crate::timefmt::now()) { + Ok(Some(x)) => x, + Ok(None) => return Err(bad_credentials()), + Err(e) => return Err(internal(app, &e)), + }; + if req.method != path.sub.method() { + return Err(method_not_allowed(path.sub.method())); + } + // After credentials and method, so the fixed error order holds for a malformed `service` too. + let service = service_of(&path.sub, req.param("service")).map_err(|NotSmart| not_found())?; + let (needs_write, perm) = match service { + Some(Service::Receive) => (true, Perm::Write), + _ => (false, Perm::Read), + }; + let need = Need( + Cat::Repository, + if needs_write { + Level::Write + } else { + Level::Read + }, + ); + if scope::public_only(&token.scopes) || !scope::allows(&token.scopes, need) { + return Err(forbidden()); + } + let access = match repos::resolve(&conn, &user, &path.owner, &path.repo) { + Ok(Some(a)) => a, + Ok(None) => return Err(repo_not_found()), + Err(e) => return Err(internal(app, &e)), + }; + if access.perm < perm { + return Err(forbidden()); + } + Ok(Plan { + user, + access, + path, + service, + may_write: needs_write, + }) +} + +fn internal(app: &App, e: &dyn std::fmt::Display) -> Resp { + eprintln!("sc-forge: internal error: {e}"); + let _ = app; + text(500, "internal server error\n") +} + +// ---- the CGI response ---- + +/// A parsed CGI response head. +#[derive(Debug, PartialEq, Eq)] +pub struct Head { + pub status: u16, + /// Headers worth forwarding (whitelisted, validated). + pub headers: Vec<(String, String)>, +} + +/// Response headers forwarded from the CGI; everything else (hop-by-hop, `Status`, anything +/// unexpected) is dropped. +const FORWARD: [&str; 7] = [ + "content-type", + "cache-control", + "expires", + "pragma", + "last-modified", + "etag", + "content-encoding", +]; + +/// Parse a CGI header block (`Name: value` lines ended by a blank line, LF or CRLF). `Status:` +/// sets the status (default 200). Returns the head and the number of bytes consumed, or `None` +/// when the block is not complete yet. `Err` when it is malformed. +pub fn parse_head(buf: &[u8]) -> Result, &'static str> { + let end = match buf + .windows(2) + .position(|w| w == b"\n\n") + .map(|i| i + 2) + .into_iter() + .chain(buf.windows(4).position(|w| w == b"\r\n\r\n").map(|i| i + 4)) + .min() + { + Some(e) => e, + None => { + return if buf.len() > 16 * 1024 { + Err("response header block too large") + } else { + Ok(None) + } + } + }; + let text = std::str::from_utf8(&buf[..end]).map_err(|_| "response headers are not UTF-8")?; + let mut head = Head { + status: 200, + headers: Vec::new(), + }; + for line in text.lines().filter(|l| !l.is_empty()) { + let (k, v) = line + .split_once(':') + .ok_or("malformed response header line")?; + let (k, v) = (k.trim().to_ascii_lowercase(), v.trim()); + if k == "status" { + head.status = v + .split_whitespace() + .next() + .and_then(|n| n.parse().ok()) + .filter(|s| (100..600).contains(s)) + .ok_or("malformed Status")?; + } else if FORWARD.contains(&k.as_str()) && !v.bytes().any(|b| b < 0x20 || b == 0x7f) { + head.headers.push((k, v.to_string())); + } + } + Ok(Some((head, end))) +} + +// ---- the CGI environment ---- + +/// `Git-Protocol` is passed to git only in the form `version=N` with optional `:key=value` parts +/// of safe characters, at most 128 bytes. +pub fn protocol_header(v: Option<&str>) -> Option { + let v = v?; + let rest = v.strip_prefix("version=")?; + let (num, tail) = rest.split_once(':').map_or((rest, ""), |(n, t)| (n, t)); + let ok = v.len() <= 128 + && !num.is_empty() + && num.bytes().all(|b| b.is_ascii_digit()) + && tail + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'=' | b':' | b'_' | b'.' | b'-')); + ok.then(|| v.to_string()) +} + +/// A request `Content-Type` is passed through only as a short printable-ASCII value. +pub fn content_type(v: Option<&str>) -> Option { + let v = v?; + (v.len() <= 128 && v.bytes().all(|b| (0x20..0x7f).contains(&b))).then(|| v.to_string()) +} + +/// Free bytes on the filesystem holding `path`. +pub fn free_bytes(path: &std::path::Path) -> std::io::Result { + use std::os::unix::ffi::OsStrExt; + let c = std::ffi::CString::new(path.as_os_str().as_bytes()) + .map_err(|_| std::io::Error::other("path contains NUL"))?; + // SAFETY: statvfs fills the zeroed struct for a valid NUL-terminated path. + let mut st: libc::statvfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok((st.f_bavail as u64).saturating_mul(st.f_frsize as u64)) +} + +/// Accounting of disk promised to pushes in flight. A reservation is accounting, not +/// enforcement: a push can still write more than it reserved (bounded by `receive.maxInputSize` +/// and `RLIMIT_FSIZE`); a hard guarantee needs filesystem quotas. +#[derive(Default)] +pub struct Reserve { + held: std::sync::Mutex, +} + +/// Bytes held for one push; returned on drop. +pub struct Held { + pool: std::sync::Arc, + bytes: u64, +} + +impl Reserve { + /// Try to hold `need` more bytes given `free` bytes on disk and a `floor` that must stay free. + fn try_hold(&self, free: u64, floor: u64, need: u64) -> bool { + let mut h = self.held.lock().unwrap_or_else(|e| e.into_inner()); + if free.saturating_sub(*h) < floor.saturating_add(need) { + return false; + } + *h += need; + true + } + + /// Reserve `need` bytes for a push, or `None` when the disk cannot promise it (507). + pub fn reserve(self: &std::sync::Arc, free: u64, floor: u64, need: u64) -> Option { + // lazy `then`: a refused reservation must not build a guard whose drop returns bytes never held + self.try_hold(free, floor, need).then(|| Held { + pool: self.clone(), + bytes: need, + }) + } + + /// Total currently promised. + pub fn held(&self) -> u64 { + *self.held.lock().unwrap_or_else(|e| e.into_inner()) + } +} + +impl Held { + /// Grow the reservation by `more` bytes as a push proceeds; `false` when the disk cannot + /// promise it (the push is aborted). + pub fn grow(&mut self, free: u64, floor: u64, more: u64) -> bool { + if self.pool.try_hold(free, floor, more) { + self.bytes += more; + true + } else { + false + } + } +} + +impl Drop for Held { + fn drop(&mut self) { + let mut h = self.pool.held.lock().unwrap_or_else(|e| e.into_inner()); + *h = h.saturating_sub(self.bytes); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn git(path: &str) -> GitPath { + match parse(path) { + Route::Git(g) => g, + other => panic!("{path}: {other:?}"), + } + } + + #[test] + fn the_whitelist_accepts_exactly_the_recorded_shapes() { + assert_eq!(git("/acme/alpha.git/info/refs").sub, Sub::InfoRefs); + assert_eq!(git("/acme/alpha.git/git-upload-pack").sub, Sub::UploadPack); + assert_eq!( + git("/acme/alpha.git/git-receive-pack").sub, + Sub::ReceivePack + ); + assert_eq!(git("/acme/alpha.git/HEAD").sub, Sub::Head); + assert_eq!( + git("/acme/alpha.git/objects/info/packs").sub, + Sub::InfoPacks + ); + let loose = format!("objects/ab/{}", "c".repeat(38)); + assert_eq!( + git(&format!("/acme/alpha.git/{loose}")).sub, + Sub::Loose(loose.clone()) + ); + assert_eq!( + git(&format!("/acme/alpha.git/{loose}")).sub.path_info(), + format!("/{loose}") + ); + let pack = format!("objects/pack/pack-{}.idx", "0".repeat(40)); + assert_eq!(git(&format!("/acme/alpha.git/{pack}")).sub, Sub::Pack(pack)); + let g = git("/Acme/Alpha-1.x.git/info/refs"); + assert_eq!((g.owner.as_str(), g.repo.as_str()), ("Acme", "Alpha-1.x")); + } + + #[test] + fn everything_else_under_a_git_path_is_unknown_and_never_reaches_the_cgi() { + for bad in [ + "/acme/alpha.git/", + "/acme/alpha.git", + "/acme/alpha.git/config", + "/acme/alpha.git/.git/config", + "/acme/alpha.git/hooks/pre-receive", + "/acme/alpha.git/info/refs/", + "/acme/alpha.git/info/refs/x", + "/acme/alpha.git//info/refs", + "/acme/alpha.git/../beta.git/info/refs", + "/acme/alpha.git/objects/info/alternates", + "/acme/alpha.git/objects/ab/short", + "/acme/alpha.git/objects/AB/cccccccccccccccccccccccccccccccccccccc", + "/acme/alpha.git/objects/ab/gggggggggggggggggggggggggggggggggggggg", + "/acme/alpha.git/objects/pack/pack-zz.pack", + "/acme/alpha.git/objects/pack/other.pack", + "/acme/alpha.git/refs/heads/main", + "/acme/alpha.git/git-upload-pack%2f..%2f", + "/acme/alpha.git/info%2Frefs", + "/acme/alpha.git/HEAD/x", + "/acme/alpha.git/git-receive-pack/", + ] { + assert_eq!(parse(bad), Route::Unknown, "{bad}"); + } + } + + #[test] + fn owner_and_repo_segments_cannot_smuggle_separators_or_traversal() { + for bad in [ + "/%2e%2e/alpha.git/info/refs", + "/acme/%2e%2e.git/info/refs", + "/acme/al%2Fpha.git/info/refs", + "/ac%00me/alpha.git/info/refs", + "/acme/alpha%0d%0a.git/info/refs", + "/a cme/alpha.git/info/refs", + "//alpha.git/info/refs", + "/acme/.git/info/refs", + "/acme/%zz.git/info/refs", + ] { + assert!( + matches!(parse(bad), Route::Unknown | Route::NotGit), + "{bad}: {:?}", + parse(bad) + ); + } + // dotted names are legal repository names, and the encoded form decodes to the same name + assert_eq!(git("/acme/.hidden.git/info/refs").repo, ".hidden"); + assert_eq!(git("/acme/a%2Db.git/info/refs").repo, "a-b"); + } + + #[test] + fn non_git_paths_fall_through_to_the_ordinary_router() { + for p in [ + "/", + "/api/v1/user", + "/acme/alpha", + "/acme", + "/acme/alpha/info/refs", + "healthz", + "/api/healthz", + ] { + assert_eq!(parse(p), Route::NotGit, "{p}"); + } + } + + #[test] + fn info_refs_needs_a_smart_service() { + assert_eq!( + service_of(&Sub::InfoRefs, Some("git-upload-pack")), + Ok(Some(Service::Upload)) + ); + assert_eq!( + service_of(&Sub::InfoRefs, Some("git-receive-pack")), + Ok(Some(Service::Receive)) + ); + assert_eq!( + service_of(&Sub::InfoRefs, None), + Err(NotSmart), + "git's dumb listing ignores hideRefs" + ); + assert_eq!( + service_of(&Sub::InfoRefs, Some("git-upload-archive")), + Err(NotSmart) + ); + assert_eq!( + service_of(&Sub::UploadPack, None), + Ok(Some(Service::Upload)) + ); + assert_eq!(service_of(&Sub::Head, None), Ok(None)); + } + + #[test] + fn methods_follow_the_sub_path() { + assert_eq!(Sub::UploadPack.method(), "POST"); + assert_eq!(Sub::ReceivePack.method(), "POST"); + assert_eq!(Sub::InfoRefs.method(), "GET"); + assert_eq!(Sub::Head.method(), "GET"); + } + + #[test] + fn every_token_form_is_extracted_and_a_password_is_just_a_token_that_will_not_verify() { + assert_eq!(credential(&Cred::Header("t".into())), Some("t")); + assert_eq!( + credential(&Cred::Basic { + user: "oauth2".into(), + pass: "t".into() + }), + Some("t") + ); + assert_eq!( + credential(&Cred::Basic { + user: "t".into(), + pass: String::new() + }), + Some("t") + ); + assert_eq!(credential(&Cred::Anonymous), None); + } + + #[test] + fn the_cgi_head_parser_reads_status_and_forwards_only_whitelisted_headers() { + let raw = b"Status: 403 Forbidden\r\nContent-Type: text/plain\r\nX-Evil: 1\r\nSet-Cookie: a=b\r\nCache-Control: no-cache\r\n\r\nbody"; + let (h, n) = parse_head(raw).unwrap().unwrap(); + assert_eq!(h.status, 403); + assert_eq!( + h.headers, + vec![ + ("content-type".to_string(), "text/plain".to_string()), + ("cache-control".to_string(), "no-cache".to_string()) + ] + ); + assert_eq!(&raw[n..], b"body"); + let (h, _) = parse_head(b"Content-Type: application/x-git-upload-pack-advertisement\nExpires: Fri, 01 Jan 1980 00:00:00 GMT\n\nx").unwrap().unwrap(); + assert_eq!( + (h.status, h.headers.len()), + (200, 2), + "bare LF and a default status of 200" + ); + assert_eq!( + parse_head(b"Content-Type: a\r\n").unwrap(), + None, + "incomplete" + ); + assert!(parse_head(b"no colon here\n\n").is_err()); + assert!(parse_head(b"Status: banana\n\n").is_err()); + assert!(parse_head(b"Status: 99\n\n").is_err()); + assert!( + parse_head(&vec![b'a'; 20_000]).is_err(), + "an unterminated header block is bounded" + ); + let (h, _) = parse_head(b"Content-Type: x\x01y\n\n").unwrap().unwrap(); + assert!( + h.headers.is_empty(), + "a header value with control characters is dropped" + ); + } + + #[test] + fn the_protocol_header_is_filtered() { + assert_eq!(protocol_header(Some("version=2")), Some("version=2".into())); + assert_eq!( + protocol_header(Some("version=2:object-format=sha1")), + Some("version=2:object-format=sha1".into()) + ); + for bad in [ + "", + "version=", + "version=x", + "version=2\r\nX: y", + "foo", + "version=2:a b", + &format!("version=2:{}", "a".repeat(200)), + ] { + assert_eq!(protocol_header(Some(bad)), None, "{bad:?}"); + } + assert_eq!(protocol_header(None), None); + } + + #[test] + fn content_types_are_short_printable_ascii() { + assert!(content_type(Some("application/x-git-upload-pack-request")).is_some()); + assert!( + content_type(Some("a\r\nb")).is_none() + && content_type(Some(&"a".repeat(200))).is_none() + && content_type(None).is_none() + ); + } + + #[test] + fn the_reservation_never_overcommits_and_returns_on_drop() { + let r = std::sync::Arc::new(Reserve::default()); + let a = r + .reserve(1000, 100, 400) + .expect("first fits: 1000 free, floor 100, need 400"); + assert_eq!(r.held(), 400); + let b = r + .reserve(1000, 100, 400) + .expect("second fits: (1000-400) >= 100+400"); + assert!( + r.reserve(1000, 100, 101).is_none(), + "1000 - 800 < 100 + 101: refuse (507), do not overcommit" + ); + assert!( + r.reserve(1000, 100, 100).is_some(), + "but exactly the remainder above the floor still fits" + ); + drop(a); + assert_eq!(r.held(), 400); + let mut c = r.reserve(1000, 100, 100).unwrap(); + assert!(c.grow(1000, 100, 300), "(1000-500) >= 100+300"); + assert!( + !c.grow(1000, 100, 300), + "the next step does not fit: the push is aborted" + ); + drop((b, c)); + assert_eq!(r.held(), 0, "everything is returned"); + } + + #[test] + fn free_space_is_readable() { + assert!(free_bytes(&std::env::temp_dir()).unwrap() > 0); + assert!(free_bytes(std::path::Path::new("/definitely/not/here")).is_err()); + } +} diff --git a/crates/forge/tests/common/mod.rs b/crates/forge/tests/common/mod.rs index 4cd53194..af09da69 100644 --- a/crates/forge/tests/common/mod.rs +++ b/crates/forge/tests/common/mod.rs @@ -24,6 +24,8 @@ pub struct Srv { pub admin: String, /// The runner the server uses (tests occupy its slots to stage a busy server). pub runner: std::sync::Arc, + /// The shared server state (tests read the transport registry). + pub app: std::sync::Arc, _data_lock: scl_forge::lifecycle::DirLock, stop: Option>, thread: Option>, @@ -87,17 +89,23 @@ impl Srv { /// Start with a runner of `slots` concurrent git children and the given admission deadline. pub fn start_git(slots: usize, admission: std::time::Duration) -> Srv { - Srv::start_full(Limits::default(), None, Some((slots, admission))) + Srv::start_full(Limits::default(), None, Some((slots, admission)), None) } fn start_cfg(limits: Limits, backoff: Option) -> Srv { - Srv::start_full(limits, backoff, None) + Srv::start_full(limits, backoff, None, None) + } + + /// Start with custom git transport limits. + pub fn start_transport(cfg: scl_forge::cgi::Cfg) -> Srv { + Srv::start_full(Limits::default(), None, None, Some(cfg)) } fn start_full( limits: Limits, backoff: Option, git_cfg: Option<(usize, std::time::Duration)>, + tcfg: Option, ) -> Srv { let dir = std::env::temp_dir().join(format!( "scl-forge-e2e-{}-{}", @@ -129,12 +137,21 @@ impl Srv { .unwrap() .with_limits(std::time::Duration::from_secs(30), admission, 16 << 20), ); + scl_forge::hook::install( + git.data_dir(), + std::path::Path::new(env!("CARGO_BIN_EXE_sc-forge")), + ) + .unwrap(); let life = scl_forge::lifecycle::Lifecycle::new(&dir, git.clone()).unwrap(); let mut app = scl_forge::routes::App::new(dir.clone(), &base, Proxies::default(), 4, life); if let Some(b) = backoff { app.backoff = b; } + if let Some(c) = tcfg { + app.transports = scl_forge::cgi::Transports::new(c); + } let app = std::sync::Arc::new(app); + let app_handle = app.clone(); let (tx, rx) = tokio::sync::oneshot::channel(); let thread = std::thread::spawn(move || { let rt = tokio::runtime::Builder::new_multi_thread() @@ -158,6 +175,7 @@ impl Srv { }); Srv { runner: git, + app: app_handle, _data_lock: data_lock, addr, dir, diff --git a/crates/forge/tests/e2e_git.rs b/crates/forge/tests/e2e_git.rs new file mode 100644 index 00000000..044c6740 --- /dev/null +++ b/crates/forge/tests/e2e_git.rs @@ -0,0 +1,1022 @@ +//! F2b: the smart-HTTP git transport, driven by the real `git` client against an in-process server. +#![cfg(unix)] + +mod common; +use common::*; +use serde_json::json; +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct Client { + dir: PathBuf, +} + +impl Client { + fn new(tag: &str) -> Client { + let dir = + std::env::temp_dir().join(format!("scl-forge-client-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + Client { dir } + } + + /// Run the client's git with a scrubbed environment; `header` is the Authorization value. + fn git_in(&self, cwd: &Path, header: Option<&str>, args: &[&str]) -> (bool, String) { + let mut cmd = Command::new("git"); + cmd.env_clear() + .env("PATH", "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin") + .env("HOME", &self.dir) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_AUTHOR_NAME", "T") + .env("GIT_AUTHOR_EMAIL", "t@example.invalid") + .env("GIT_COMMITTER_NAME", "T") + .env("GIT_COMMITTER_EMAIL", "t@example.invalid") + .current_dir(cwd); + if let Some(h) = header { + cmd.arg("-c") + .arg(format!("http.extraHeader=Authorization: {h}")); + } + let out = cmd.args(args).output().expect("run git"); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out.status.success(), text) + } + + fn ok(&self, cwd: &Path, header: Option<&str>, args: &[&str]) -> String { + let (ok, text) = self.git_in(cwd, header, args); + assert!(ok, "git {args:?}: {text}"); + text + } +} + +impl Drop for Client { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn repo(s: &Srv, name: &str) -> String { + let r = s.post( + "/api/v1/user/repos", + &token(&s.admin), + json!({"name": name, "default_branch": "main"}), + ); + assert_eq!(r.status, 201, "{}", r.msg()); + format!("{}/siteadmin/{name}.git", s.base) +} + +#[test] +fn push_then_clone_round_trips() { + let s = Srv::start(); + let c = Client::new("rt"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let work = c.dir.join("work"); + c.ok( + &c.dir, + None, + &["init", "-q", "-b", "main", work.to_str().unwrap()], + ); + std::fs::write(work.join("a.txt"), "hello\n").unwrap(); + c.ok(&work, None, &["add", "."]); + c.ok(&work, None, &["commit", "-q", "-m", "one"]); + c.ok(&work, Some(&auth), &["push", &url, "main"]); + let clone = c.dir.join("clone"); + c.ok( + &c.dir, + Some(&auth), + &["clone", "-q", &url, clone.to_str().unwrap()], + ); + assert_eq!( + std::fs::read_to_string(clone.join("a.txt")).unwrap(), + "hello\n" + ); +} + +// ---- helpers ---- + +fn put_commit(c: &Client, work: &Path, file: &str, text: &str) { + std::fs::write(work.join(file), text).unwrap(); + c.ok(work, None, &["add", "."]); + c.ok(work, None, &["commit", "-q", "-m", file]); +} + +/// A working copy with one commit on `main`, pushed to `url` with `auth`. +fn seeded(c: &Client, url: &str, auth: &str) -> PathBuf { + let work = c.dir.join("work"); + c.ok( + &c.dir, + None, + &["init", "-q", "-b", "main", work.to_str().unwrap()], + ); + put_commit(c, &work, "a.txt", "hello\n"); + c.ok(&work, Some(auth), &["push", "-q", url, "main"]); + work +} + +fn body_text(r: &Resp) -> String { + String::from_utf8_lossy(&r.raw).to_string() +} + +const UP: &str = "/info/refs?service=git-upload-pack"; +const RP: &str = "/info/refs?service=git-receive-pack"; + +// ---- credentials and answers ---- + +#[test] +fn every_credential_form_is_accepted_and_each_failure_has_its_recorded_text() { + let s = Srv::start(); + let t = s.admin.clone(); + repo(&s, "smart"); + let path = format!("/siteadmin/smart.git{UP}"); + + let anon = s.send("GET", &path, None, None); + assert_eq!( + (anon.status, body_text(&anon).as_str()), + (401, "Unauthorized\n") + ); + assert!(anon.headers["www-authenticate"].starts_with("Basic realm=")); + + for good in [ + token(&t), + format!("Bearer {t}"), + basic("anyone", &t), + basic("oauth2", &t), + basic(&t, ""), + ] { + let r = s.send("GET", &path, Some(&good), None); + assert_eq!(r.status, 200, "{good}: {}", body_text(&r)); + assert_eq!( + r.headers["content-type"], + "application/x-git-upload-pack-advertisement" + ); + // (the harness does not decode the chunked framing) + assert!( + body_text(&r).contains("001e# service=git-upload-pack\n"), + "{good}: {:?}", + body_text(&r) + ); + } + + for bad in [ + token("nonsense"), + basic("siteadmin", "nonsense"), + format!("Bearer {}", "0".repeat(40)), + ] { + let r = s.send("GET", &path, Some(&bad), None); + assert_eq!( + (r.status, body_text(&r).as_str()), + (401, "Verify\n"), + "{bad}" + ); + } +} + +#[test] +fn a_password_is_never_a_git_credential() { + let s = Srv::start(); + let pw = s.mk_user("alice"); + repo(&s, "smart"); + let r = s.send( + "GET", + &format!("/siteadmin/smart.git{UP}"), + Some(&basic("alice", &pw)), + None, + ); + assert_eq!((r.status, body_text(&r).as_str()), (401, "Verify\n")); +} + +#[test] +fn missing_and_invisible_repositories_answer_identically() { + let s = Srv::start(); + let pw = s.mk_user("bob"); + let bob = token(&s.mk_token("bob", &pw, "t", &["all"])); + let r = s.post( + "/api/v1/user/repos", + &token(&s.admin), + json!({"name": "secret", "private": true}), + ); + assert_eq!(r.status, 201); + for repo in ["secret", "nothere"] { + for q in [UP, RP] { + let r = s.send( + "GET", + &format!("/siteadmin/{repo}.git{q}"), + Some(&bob), + None, + ); + assert_eq!( + (r.status, body_text(&r).as_str()), + (404, "Repository not found"), + "{repo}{q}" + ); + } + } +} + +#[test] +fn scopes_and_permissions_gate_each_service() { + let s = Srv::start(); + let (pw_r, pw_w) = (s.mk_user("reader"), s.mk_user("writer")); + let reader = s.mk_token("reader", &pw_r, "t", &["all"]); + repo(&s, "smart"); + let put = |user: &str, mode: &str| { + let r = s.send( + "PUT", + &format!("/api/v1/repos/siteadmin/smart/collaborators/{user}"), + Some(&token(&s.admin)), + Some(&json!({"permission": mode})), + ); + assert_eq!(r.status, 204, "{}", r.msg()); + }; + put("reader", "read"); + put("writer", "write"); + let writer = s.mk_token("writer", &pw_w, "t", &["all"]); + let code = |auth: &str, q: &str| { + s.send( + "GET", + &format!("/siteadmin/smart.git{q}"), + Some(&token(auth)), + None, + ) + .status + }; + assert_eq!(code(&reader, UP), 200); + assert_eq!(code(&reader, RP), 403, "read collaborator cannot push"); + assert_eq!(code(&writer, RP), 200); + // Scope: a read-only token reads but does not push even where the user may write. + let ro = s.mk_token("writer", &pw_w, "ro", &["read:repository"]); + assert_eq!(code(&ro, UP), 200); + assert_eq!(code(&ro, RP), 403); + let other = s.mk_token("writer", &pw_w, "other", &["read:user"]); + assert_eq!(code(&other, UP), 403, "no repository scope at all"); + let public = s.mk_token("writer", &pw_w, "pub", &["public-only", "read:repository"]); + assert_eq!( + code(&public, UP), + 403, + "public-only tokens never reach repository transport" + ); +} + +#[test] +fn only_the_whitelisted_shapes_and_methods_are_served() { + let s = Srv::start(); + let a = token(&s.admin); + repo(&s, "smart"); + let get = |p: &str| { + s.send("GET", &format!("/siteadmin/smart.git{p}"), Some(&a), None) + .status + }; + for p in [ + "/config", + "/hooks/pre-receive", + "/refs/heads/main", + "/objects/../config", + "/info/refs", + "/info/refs?service=git-bogus", + "/objects/info/alternates", + "/forge", + "/../x", + ] { + assert_eq!(get(p), 404, "{p}"); + } + assert_eq!(get("/HEAD"), 200); + let post = |p: &str| { + s.send("POST", &format!("/siteadmin/smart.git{p}"), Some(&a), None) + .status + }; + assert_eq!(post("/info/refs?service=git-upload-pack"), 405); + assert_eq!( + get("/git-receive-pack"), + 405, + "an RPC endpoint is POST only" + ); + assert_eq!(post("/HEAD"), 405); + // Encoded path separators and dots cannot name anything else. + for p in [ + "/siteadmin/%2e%2e.git/info/refs", + "/siteadmin%2Fsmart.git/info/refs", + "/siteadmin/smart%2egit/info/refs", + ] { + let r = s.send("GET", p, Some(&a), None); + assert!(r.status == 404, "{p}: {}", r.status); + } +} + +// ---- real client flows ---- + +#[test] +fn branches_tags_force_and_deletes_by_push() { + let s = Srv::start(); + let c = Client::new("flows"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let work = seeded(&c, &url, &auth); + c.ok( + &work, + Some(&auth), + &["push", "-q", &url, "main:refs/heads/dev"], + ); + c.ok(&work, None, &["tag", "v1"]); + c.ok(&work, Some(&auth), &["push", "-q", &url, "v1"]); + put_commit(&c, &work, "b.txt", "two\n"); + c.ok(&work, Some(&auth), &["push", "-q", &url, "main"]); + // forced rewind of dev, then a stale lease is refused + c.ok( + &work, + Some(&auth), + &["push", "-q", "-f", &url, "main:refs/heads/dev"], + ); + let (ok, _) = c.git_in( + &work, + Some(&auth), + &[ + "push", + "-q", + "--force-with-lease=dev:0000000000000000000000000000000000000001", + &url, + "main~1:refs/heads/dev", + ], + ); + assert!(!ok, "stale lease must be refused"); + c.ok( + &work, + Some(&auth), + &["push", "-q", &url, ":refs/heads/dev", ":refs/tags/v1"], + ); + let refs = c.ok(&work, Some(&auth), &["ls-remote", &url]); + assert!( + refs.contains("refs/heads/main") && !refs.contains("dev") && !refs.contains("v1"), + "{refs}" + ); + // the push stamped the repository + let r = s.get("/api/v1/repos/siteadmin/smart", &auth); + assert_eq!(r.status, 200); + assert_eq!(r.json()["empty"], false); +} + +#[test] +fn the_ref_policy_is_enforced_for_every_unwritable_namespace_and_the_default_branch() { + let s = Srv::start(); + let c = Client::new("policy"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let work = seeded(&c, &url, &auth); + for refname in [ + "refs/replace/abc", + "refs/pull/1/head", + "refs/notes/commits", + "refs/changes/01/1/1", + "refs/forge/x", + "refs/stash", + ] { + let (ok, text) = c.git_in( + &work, + Some(&auth), + &["push", &url, &format!("main:{refname}")], + ); + assert!(!ok, "{refname} must be refused: {text}"); + } + let (ok, text) = c.git_in(&work, Some(&auth), &["push", &url, ":refs/heads/main"]); + assert!( + !ok && text.contains("pre-receive hook declined") || text.contains("default branch"), + "{text}" + ); + let refs = c.ok(&work, Some(&auth), &["ls-remote", &url]); + assert!( + refs.contains("refs/heads/main"), + "the default branch survives: {refs}" + ); + assert!( + !refs.contains("replace") && !refs.contains("pull") && !refs.contains("notes"), + "{refs}" + ); +} + +#[test] +fn refs_forge_is_hidden_from_both_services_even_when_it_exists() { + let s = Srv::start(); + let c = Client::new("hidden"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let work = seeded(&c, &url, &auth); + let oid = s.git(1, &["rev-parse", "refs/heads/main"], None); + s.git(1, &["update-ref", "refs/forge/internal", &oid], None); + for q in [UP, RP] { + let r = s.send( + "GET", + &format!("/siteadmin/smart.git{q}"), + Some(&auth), + None, + ); + assert!(!body_text(&r).contains("refs/forge"), "{q}"); + } + let (ok, _) = c.git_in(&work, Some(&auth), &["fetch", &url, "refs/forge/internal"]); + assert!(!ok, "a hidden ref cannot be fetched by name"); +} + +#[test] +fn fetch_by_sha_filters_shallow_and_both_protocol_versions_work() { + let s = Srv::start(); + let c = Client::new("fetch"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let work = seeded(&c, &url, &auth); + put_commit(&c, &work, "b.txt", "two\n"); + c.ok(&work, Some(&auth), &["push", "-q", &url, "main"]); + let tip = c.ok(&work, None, &["rev-parse", "HEAD"]).trim().to_string(); + for v in ["0", "2"] { + let dst = c.dir.join(format!("c{v}")); + c.ok( + &c.dir, + Some(&auth), + &[ + "-c", + &format!("protocol.version={v}"), + "clone", + "-q", + &url, + dst.to_str().unwrap(), + ], + ); + assert_eq!(c.ok(&dst, None, &["rev-parse", "HEAD"]).trim(), tip, "v{v}"); + } + let lite = c.dir.join("lite"); + c.ok( + &c.dir, + Some(&auth), + &[ + "clone", + "-q", + "--filter=blob:none", + &url, + lite.to_str().unwrap(), + ], + ); + assert!(c + .ok(&lite, None, &["config", "remote.origin.partialclonefilter"]) + .contains("blob:none")); + let shallow = c.dir.join("shallow"); + c.ok( + &c.dir, + Some(&auth), + &["clone", "-q", "--depth=1", &url, shallow.to_str().unwrap()], + ); + assert_eq!( + c.ok(&shallow, None, &["rev-list", "--count", "HEAD"]) + .trim(), + "1" + ); + c.ok(&shallow, Some(&auth), &["fetch", "-q", "--unshallow"]); + assert_eq!( + c.ok(&shallow, None, &["rev-list", "--count", "HEAD"]) + .trim(), + "2" + ); + // fetch an unreferenced-by-branch commit by its id (allowAnySHA1InWant) + c.ok( + &work, + Some(&auth), + &["push", "-q", &url, "main~1:refs/heads/old"], + ); + let old = c + .ok(&work, None, &["rev-parse", "main~1"]) + .trim() + .to_string(); + c.ok(&work, Some(&auth), &["push", "-q", &url, ":refs/heads/old"]); + let by = c.dir.join("by"); + c.ok(&c.dir, None, &["init", "-q", by.to_str().unwrap()]); + c.ok(&by, Some(&auth), &["fetch", "-q", &url, &old]); +} + +#[test] +fn gzip_request_bodies_are_inflated_by_the_server_under_a_cap() { + use std::io::Write; + let s = Srv::start_transport(scl_forge::cgi::Cfg { + max_request: 1 << 20, + ..Default::default() + }); + let c = Client::new("gzip"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + seeded(&c, &url, &auth); + let gz = |data: &[u8]| { + let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(data).unwrap(); + e.finish().unwrap() + }; + let post = |body: &[u8], enc: &str| { + let head = format!( + "POST /siteadmin/smart.git/git-upload-pack HTTP/1.1\r\nHost: x\r\nConnection: close\r\nAuthorization: {auth}\r\nGit-Protocol: version=2\r\nContent-Type: application/x-git-upload-pack-request\r\nContent-Encoding: {enc}\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + s.raw(head.as_bytes(), body) + }; + let ls_refs = b"0014command=ls-refs\n00010009peel\n0000"; + let r = post(&gz(ls_refs), "gzip"); + assert_eq!(r.status, 200, "{}", body_text(&r)); + assert!(body_text(&r).contains("refs/heads/main")); + // a bomb: tiny on the wire, far over the inflated cap + let bomb = gz(&vec![b'0'; 64 << 20]); + assert!(bomb.len() < 200_000); + assert_eq!(post(&bomb, "gzip").status, 413); + assert_eq!(post(b"not gzip at all", "gzip").status, 400); + assert_eq!(post(ls_refs, "br").status, 415); + // git never gzips a push; the server refuses it rather than guess + let head = format!( + "POST /siteadmin/smart.git/git-receive-pack HTTP/1.1\r\nHost: x\r\nConnection: close\r\nAuthorization: {auth}\r\nContent-Encoding: gzip\r\nContent-Length: 3\r\n\r\n" + ); + assert_eq!(s.raw(head.as_bytes(), b"abc").status, 415); +} + +// ---- limits, admission, process control ---- + +/// Open a push the client never finishes: the transport stays in flight until the socket closes. +fn stalled_push(s: &Srv, auth: &str, repo: &str) -> std::net::TcpStream { + use std::io::Write; + let mut sock = std::net::TcpStream::connect(s.addr).unwrap(); + sock.set_read_timeout(Some(std::time::Duration::from_secs(10))) + .unwrap(); + let head = format!( + "POST /siteadmin/{repo}.git/git-receive-pack HTTP/1.1\r\nHost: x\r\nAuthorization: {auth}\r\nContent-Type: application/x-git-receive-pack-request\r\nContent-Length: 1000\r\n\r\n0000" + ); + sock.write_all(head.as_bytes()).unwrap(); + sock +} + +fn wait_for(what: &str, mut f: impl FnMut() -> bool) { + let end = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !f() { + assert!( + std::time::Instant::now() < end, + "timed out waiting for {what}" + ); + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +#[test] +fn slots_are_bounded_then_released_when_the_client_disconnects() { + let s = Srv::start_transport(scl_forge::cgi::Cfg { + max_transports: 1, + admission_wait: std::time::Duration::from_millis(200), + ..Default::default() + }); + let auth = token(&s.admin); + repo(&s, "smart"); + let held = stalled_push(&s, &auth, "smart"); + wait_for("the transport to register", || { + s.app.life.registry.live() == 1 + }); + let r = s.send( + "GET", + &format!("/siteadmin/smart.git{UP}"), + Some(&auth), + None, + ); + assert_eq!(r.status, 503, "{}", body_text(&r)); + assert_eq!(r.headers["retry-after"], "5"); + drop(held); + wait_for("the child to be reaped after the disconnect", || { + s.app.life.registry.live() == 0 + }); + wait_for("the slot to come back", || { + s.send( + "GET", + &format!("/siteadmin/smart.git{UP}"), + Some(&auth), + None, + ) + .status + == 200 + }); +} + +#[test] +fn a_stalled_upload_is_cut_off_by_the_idle_deadline() { + let s = Srv::start_transport(scl_forge::cgi::Cfg { + idle: std::time::Duration::from_millis(300), + ..Default::default() + }); + let auth = token(&s.admin); + repo(&s, "smart"); + let mut sock = stalled_push(&s, &auth, "smart"); + let mut buf = Vec::new(); + use std::io::Read; + let _ = sock.read_to_end(&mut buf); + // git writes its response head before it reads the request, so the status line is already + // out; what the client must see is a connection that ends, not a clean finish. + assert!( + !String::from_utf8_lossy(&buf).contains("\r\n0\r\n\r\n"), + "the chunked body must not be terminated cleanly: {}", + String::from_utf8_lossy(&buf) + ); + wait_for("cleanup", || s.app.life.registry.live() == 0); +} + +#[test] +fn the_wall_clock_limit_ends_a_transport() { + let s = Srv::start_transport(scl_forge::cgi::Cfg { + timeout: std::time::Duration::from_millis(400), + ..Default::default() + }); + let auth = token(&s.admin); + repo(&s, "smart"); + let mut sock = stalled_push(&s, &auth, "smart"); + let mut buf = Vec::new(); + use std::io::Read; + let _ = sock.read_to_end(&mut buf); + assert!( + !String::from_utf8_lossy(&buf).contains("\r\n0\r\n\r\n"), + "{}", + String::from_utf8_lossy(&buf) + ); + wait_for("cleanup", || s.app.life.registry.live() == 0); +} + +#[test] +fn deleting_a_repository_ends_its_running_transports_first() { + let s = Srv::start(); + let auth = token(&s.admin); + repo(&s, "smart"); + let _held = stalled_push(&s, &auth, "smart"); + wait_for("the transport", || s.app.life.registry.live() == 1); + let r = s.send("DELETE", "/api/v1/repos/siteadmin/smart", Some(&auth), None); + assert_eq!(r.status, 204, "{}", r.msg()); + assert_eq!(s.app.life.registry.live(), 0); + assert!(!s.dir.join("repos").join("1.git").exists()); + assert!(!s.dir.join("trash").join("1.git").exists()); + let r = s.send( + "GET", + &format!("/siteadmin/smart.git{UP}"), + Some(&auth), + None, + ); + assert_eq!(r.status, 404); +} + +#[test] +fn a_push_over_the_size_cap_fails_and_changes_nothing() { + let s = Srv::start_transport(scl_forge::cgi::Cfg { + max_pack: 200 << 10, + ..Default::default() + }); + let c = Client::new("cap"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let work = seeded(&c, &url, &auth); + // incompressible bytes, so the pack really is large + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + let blob: Vec = (0..(1u32 << 20)) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 24) as u8 + }) + .collect(); + std::fs::write(work.join("big.bin"), &blob).unwrap(); + c.ok(&work, None, &["add", "."]); + c.ok(&work, None, &["commit", "-q", "-m", "big"]); + let (ok, _) = c.git_in(&work, Some(&auth), &["push", "-q", &url, "main"]); + assert!(!ok); + let before = c.ok(&work, Some(&auth), &["ls-remote", &url, "refs/heads/main"]); + let local_parent = c.ok(&work, None, &["rev-parse", "HEAD~1"]); + assert!( + before.starts_with(local_parent.trim()), + "{before} vs {local_parent}" + ); + wait_for("cleanup", || s.app.life.registry.live() == 0); +} + +#[test] +fn a_full_disk_reserve_refuses_pushes_with_507_but_not_reads() { + let s = Srv::start_transport(scl_forge::cgi::Cfg { + min_free: u64::MAX / 2, + ..Default::default() + }); + let auth = token(&s.admin); + repo(&s, "smart"); + let post = format!( + "POST /siteadmin/smart.git/git-receive-pack HTTP/1.1\r\nHost: x\r\nConnection: close\r\nAuthorization: {auth}\r\nContent-Length: 4\r\n\r\n0000" + ); + assert_eq!(s.raw(post.as_bytes(), b"").status, 507); + assert_eq!( + s.send( + "GET", + &format!("/siteadmin/smart.git{UP}"), + Some(&auth), + None + ) + .status, + 200 + ); +} + +#[test] +fn the_hook_binary_fails_closed() { + let s = Srv::start(); + repo(&s, "smart"); + let hook = s.dir.join("hooks").join("pre-receive"); + let z = "0".repeat(40); + let one = "1".repeat(40); + let run = |id: &str, dir: &Path, stdin: &str| { + use std::io::Write; + let mut child = Command::new(&hook) + .env_clear() + .env("FORGE_DATA_DIR", dir) + .env("FORGE_REPO_ID", id) + .stdin(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(stdin.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + ( + out.status.code(), + String::from_utf8_lossy(&out.stderr).to_string(), + ) + }; + let ok_line = format!("{z} {one} refs/heads/feature\n"); + assert_eq!(run("1", &s.dir, &ok_line).0, Some(0)); + for (what, id, dir, line) in [ + ( + "replace ref", + "1", + &s.dir, + format!("{z} {one} refs/replace/x\n"), + ), + ( + "default delete", + "1", + &s.dir, + format!("{one} {z} refs/heads/main\n"), + ), + ( + "default delete, other case", + "1", + &s.dir, + format!("{one} {z} refs/heads/MAIN\n"), + ), + ("non-utf8 ref", "1", &s.dir, "x".to_string()), + ("unknown repo", "999", &s.dir, ok_line.clone()), + ("missing db", "1", &s.dir.join("nowhere"), ok_line.clone()), + ] { + let (code, err) = run(id, dir, &line); + assert_eq!(code, Some(1), "{what}: {err}"); + } + assert_eq!(run("x", &s.dir, &ok_line).0, Some(1), "bad id"); +} + +// ---- streaming and process control, continued ---- + +/// 20 MiB that does not compress. +fn noise(len: usize) -> Vec { + let mut x: u64 = 0x2545_F491_4F6C_DD1D; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 24) as u8 + }) + .collect() +} + +#[test] +fn a_large_push_and_clone_stream_and_a_vanished_client_ends_the_child() { + use std::io::{Read, Write}; + let s = Srv::start(); + let c = Client::new("big"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let work = seeded(&c, &url, &auth); + std::fs::write(work.join("big.bin"), noise(20 << 20)).unwrap(); + c.ok(&work, None, &["add", "."]); + c.ok(&work, None, &["commit", "-q", "-m", "big"]); + c.ok(&work, Some(&auth), &["push", "-q", &url, "main"]); + let tip = c.ok(&work, None, &["rev-parse", "HEAD"]).trim().to_string(); + let dst = c.dir.join("dst"); + c.ok( + &c.dir, + Some(&auth), + &["clone", "-q", &url, dst.to_str().unwrap()], + ); + assert_eq!( + std::fs::metadata(dst.join("big.bin")).unwrap().len(), + 20 << 20 + ); + + // Ask for the whole pack (protocol v0) and hang up after the first bytes. + let want = format!("want {tip} ofs-delta\n"); + let body = format!("{:04x}{want}0000{:04x}done\n", want.len() + 4, 9); + let head = format!( + "POST /siteadmin/smart.git/git-upload-pack HTTP/1.1\r\nHost: x\r\nAuthorization: {auth}\r\nContent-Type: application/x-git-upload-pack-request\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + let mut sock = std::net::TcpStream::connect(s.addr).unwrap(); + sock.set_read_timeout(Some(std::time::Duration::from_secs(10))) + .unwrap(); + sock.write_all(head.as_bytes()).unwrap(); + sock.write_all(body.as_bytes()).unwrap(); + let mut first = [0u8; 4096]; + let n = sock.read(&mut first).unwrap(); + assert!(String::from_utf8_lossy(&first[..n]).starts_with("HTTP/1.1 200")); + drop(sock); + wait_for("the child to be ended after the hang-up", || { + s.app.life.registry.live() == 0 + }); + // and the server still serves + assert_eq!( + s.send( + "GET", + &format!("/siteadmin/smart.git{UP}"), + Some(&auth), + None + ) + .status, + 200 + ); +} + +#[test] +fn concurrent_pushes_to_one_new_branch_let_exactly_one_win() { + let s = Srv::start(); + let c = Client::new("race"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + seeded(&c, &url, &auth); + let mk = |name: &str| { + let w = c.dir.join(name); + c.ok( + &c.dir, + None, + &["init", "-q", "-b", "x", w.to_str().unwrap()], + ); + put_commit(&c, &w, &format!("{name}.txt"), name); + w + }; + let (a, b) = (mk("a"), mk("b")); + let results: Vec = std::thread::scope(|sc| { + let hs: Vec<_> = [&a, &b] + .into_iter() + .map(|w| { + sc.spawn(|| { + c.git_in(w, Some(&auth), &["push", "-q", &url, "x:refs/heads/race"]) + .0 + }) + }) + .collect(); + hs.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert_eq!(results.iter().filter(|ok| **ok).count(), 1, "{results:?}"); + s.git(1, &["fsck", "--no-progress"], None); +} + +#[test] +fn shutdown_ends_a_running_transport() { + let s = Srv::start(); + let auth = token(&s.admin); + repo(&s, "smart"); + let held = stalled_push(&s, &auth, "smart"); + wait_for("the transport", || s.app.life.registry.live() == 1); + let app = s.app.clone(); + drop(s); // stops the server: the drain deadline passes, then stragglers are cancelled + assert_eq!(app.life.registry.live(), 0); + drop(held); +} + +#[test] +fn authenticated_object_reads_are_never_publicly_cacheable() { + let s = Srv::start(); + let c = Client::new("cache"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + seeded(&c, &url, &auth); + let oid = s.git(1, &["rev-parse", "HEAD"], None); + // a loose object (git says `public, max-age=31536000` for these) and the info/refs advertisement + let loose = format!("/siteadmin/smart.git/objects/{}/{}", &oid[..2], &oid[2..]); + let r = s.send("GET", &loose, Some(&auth), None); + assert_eq!( + r.status, + 200, + "a small push leaves loose objects: {}", + body_text(&r) + ); + let cc = r.headers["cache-control"].to_ascii_lowercase(); + assert!(cc.contains("private") && !cc.contains("public"), "{cc}"); + assert!(!r.headers.contains_key("expires"), "no far-future expiry"); + let r = s.send( + "GET", + &format!("/siteadmin/smart.git{UP}"), + Some(&auth), + None, + ); + assert!(!r.headers["cache-control"] + .to_ascii_lowercase() + .contains("public")); +} + +#[test] +fn a_request_queued_for_a_slot_does_not_start_git_after_shutdown() { + let s = Srv::start_transport(scl_forge::cgi::Cfg { + max_transports: 1, + ..Default::default() + }); + let auth = token(&s.admin); + repo(&s, "smart"); + let held = stalled_push(&s, &auth, "smart"); + wait_for("the transport", || s.app.life.registry.live() == 1); + let app = s.app.clone(); + let addr = s.addr; + let admin = s.admin.clone(); + let queued = std::thread::spawn(move || { + use std::io::{Read, Write}; + let mut sock = std::net::TcpStream::connect(addr).unwrap(); + sock.set_read_timeout(Some(std::time::Duration::from_secs(20))) + .unwrap(); + write!( + sock, + "GET /siteadmin/smart.git{UP} HTTP/1.1\r\nHost: x\r\nConnection: close\r\nAuthorization: {}\r\n\r\n", + token(&admin) + ) + .ok(); + let mut buf = Vec::new(); + let _ = sock.read_to_end(&mut buf); + String::from_utf8_lossy(&buf).to_string() + }); + std::thread::sleep(std::time::Duration::from_millis(300)); // let it reach the admission queue + drop(s); + let reply = queued.join().unwrap(); + // Either the 503 reaches the client, or the runtime is already gone and the connection just + // closes; what must never happen is git starting and answering. + assert!( + reply.is_empty() || reply.starts_with("HTTP/1.1 503"), + "{reply}" + ); + assert_eq!(app.life.registry.live(), 0); + drop(held); +} + +#[test] +fn only_a_push_the_hook_accepted_stamps_the_repository() { + let s = Srv::start(); + let c = Client::new("stamp"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let work = seeded(&c, &url, &auth); + let stamp = || s.get("/api/v1/repos/siteadmin/smart", &auth).json()["updated_at"].to_string(); + wait_for("the first push to finish stamping", || { + s.app.life.registry.live() == 0 + }); + let t0 = stamp(); + // timestamps have second resolution + std::thread::sleep(std::time::Duration::from_millis(1200)); + // an advertisement and a push the hook rejects (git still exits 0 after reporting `ng`) + s.send( + "GET", + &format!("/siteadmin/smart.git{RP}"), + Some(&auth), + None, + ); + let (ok, _) = c.git_in(&work, Some(&auth), &["push", &url, "main:refs/replace/x"]); + assert!(!ok); + wait_for("cleanup", || s.app.life.registry.live() == 0); + assert_eq!(stamp(), t0, "neither stamps"); + put_commit(&c, &work, "b.txt", "two\n"); + c.ok(&work, Some(&auth), &["push", "-q", &url, "main"]); + wait_for("the stamp", || stamp() != t0); +} + +#[test] +fn a_malformed_service_keeps_the_fixed_error_order() { + let s = Srv::start(); + let auth = token(&s.admin); + repo(&s, "smart"); + let p = "/siteadmin/smart.git/info/refs?service=bogus"; + assert_eq!( + s.send("GET", p, None, None).status, + 401, + "credentials first" + ); + assert_eq!( + s.send("POST", p, Some(&auth), None).status, + 405, + "then the method" + ); + assert_eq!( + s.send("GET", p, Some(&auth), None).status, + 404, + "then the dumb listing is refused" + ); +} diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index bc4cdfa5..00b0bb34 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -837,6 +837,111 @@ protected paths, plaintext. - The ceiling (1 GiB) is compile-time, so no configuration can make a length claim unbounded. +## sc-forge: authentication, tenancy, transport, resources — P38 F1/F2 + +`sc-forge serve` is a separate trust domain from the `.sc/` features above: hosted repositories are plain +Git, there is no per-file encryption, and the boundary is the server's authentication and its process +isolation. This is threat-model pass #1 (auth, tenancy, transport, resources); passes on the content API +and on merge/protection come with F3 and F4. Each line names the test that pins it (`crates/forge/tests/e2e_git.rs` +unless said otherwise). + +- **Credentials.** Git transport accepts access tokens only, in every recorded form (`token`/`Bearer`, Basic with + the token as the password under any user name, or as the user name with an empty password); a user password is + never a git credential (`a_password_is_never_a_git_credential`). Failures answer the recorded texts: + 401 `Unauthorized`/`Verify`, 403 `Forbidden`, 404 `Repository not found` + (`every_credential_form_is_accepted_and_each_failure_has_its_recorded_text`). Authorization is evaluated once, at + the start of the request: revoking a token or a collaborator does not cancel a push already running (it is bounded + by `--git-timeout`, 30 minutes by default). +- **Existence oracles.** A repository the caller may not see, one that does not exist, and one being deleted give the + same 404 body on both services (`missing_and_invisible_repositories_answer_identically`); a token restricted to + public repositories is refused before the repository is looked up, and a read-only collaborator is refused at the + `receive-pack` advertisement (`scopes_and_permissions_gate_each_service`). +- **Tenant isolation.** The directory a transport can reach is built from the integer repository id + (`GIT_PROJECT_ROOT=/repos`, `PATH_INFO=/.git/...`), never from client text, and git is told the repository + explicitly (no discovery). Owner and repository segments must be plain name characters after percent-decoding + (`.`/`..` are refused); the rest of the path must match a whitelist byte for byte, and the sub-path handed to the CGI + is rebuilt from it (`only_the_whitelisted_shapes_and_methods_are_served`, `transport::tests`). `.git/config`, hooks, + `objects/info/alternates` and every other file are 404. Objects of a deleted branch stay readable by anyone with read + access to *that* repository (`uploadpack.allowAnySHA1InWant=true` is what makes fetch by id work; it is not a + cross-tenant read, since each repository is a separate object store and no alternates exist). +- **Ref namespace.** Only `refs/heads/*` and `refs/tags/*` are writable: `refs/replace/*`, `refs/pull/*`, `refs/notes/*`, + `refs/changes/*` and everything else are refused by the `pre-receive` hook + (`the_ref_policy_is_enforced_for_every_unwritable_namespace_and_the_default_branch`); `refs/forge/` (reserved for the + server) is hidden from both services (`transfer.hideRefs`) and cannot be fetched by name + (`refs_forge_is_hidden_from_both_services_even_when_it_exists`); replace refs are ignored on every spawn + (`GIT_NO_REPLACE_OBJECTS`, `core.useReplaceRefs=false`). The default branch cannot be deleted, compared by Unicode + lower case so a case alias does not slip past (`policy::tests`; the hook binary test + `the_hook_binary_fails_closed`). Known gap: NFC/NFD-equivalent spellings of a non-ASCII default branch are not + folded (a macOS-filesystem quirk; marked `shortcut:` in `policy.rs`). +- **Hook integrity.** `core.hooksPath` is pinned on every spawn to `/hooks`, whose `pre-receive` is a symlink to + the `sc-forge` binary, re-pointed at every start; repositories are created with `--template=` and carry no hooks of + their own. The hook reads the database through a read-only opener (no create, `query_only`, schema version checked), + never takes the data-directory lock, and **fails closed**: a database error, a missing repository, a repository that + is not `live`, a malformed or non-UTF-8 update line, or a missing environment refuses the push + (`the_hook_binary_fails_closed`, `hook::tests`). Authorization of branch protection (F4) plugs into the same + `policy::check_update` function. +- **Environment and config injection.** The CGI environment is built from scratch (`env_clear`): fixed `PATH`, a scratch + `HOME`, `GIT_CONFIG_NOSYSTEM`, `GIT_CONFIG_GLOBAL=/dev/null`, per-spawn config through `GIT_CONFIG_COUNT`. Of the + request only these reach the child: a rebuilt `PATH_INFO`, the method, `service=` rebuilt from the parsed service, + `Content-Length`, a printable `Content-Type` of at most 128 bytes, and `Git-Protocol` only when it matches + `version=N[:key=value...]` of safe characters (`transport::tests`). `REMOTE_USER` is set only after the write check, so + `http-backend` enables `receive-pack` for that request alone. The advertisement carries the server's git + version and platform (`agent=git/2.x-`), as stock git does. +- **Request framing.** Plain request bodies are never buffered: they stream through a bounded channel to the child. A + gzip `upload-pack` request is read (within 5 minutes in total) and inflated **by the server before any process starts**, under a cap on both the + compressed and the *inflated* size (`--max-request-bytes`, 16 MiB), because `git http-backend` has no counter on + expanded bytes; the child gets the plain bytes with an exact `Content-Length` and never sees `Content-Encoding`. A bomb + or bad gzip is therefore a deterministic 413/400 and never gets a process + (`gzip_request_bodies_are_inflated_by_the_server_under_a_cap`; other encodings and gzip on a push are 415). Server + memory is bounded per transport by about twice the cap while a gzip body is inflated (the buffer plus its chunked + copy) and a few fixed channel buffers otherwise; with `--max-transports` 64 the worst case is the product. +- **Cache policy.** Everything served is authenticated, so nothing may be cached publicly: git's `public, + max-age=31536000` on loose objects and packs is replaced with Gitea's private policy and its `Expires` dropped, and RPC + results and 401s carry the same private policy (`authenticated_object_reads_are_never_publicly_cacheable`). +- **Resource exhaustion.** Admission before any process exists: a per-repository slot, then a global slot, both under one + `--admission-wait` deadline (503 + `Retry-After`; per-repository first so a flood on one repository cannot hold global + slots while it waits). Pushes reserve disk (`--min-free-bytes` floor, `--push-reserve` up front, grown in 64 MiB + steps as bytes arrive); an unreservable push is 507 before it starts, or aborted mid-stream. The reservation is + accounting, not enforcement: a push can write more than it reserved up to `--max-pack-size` + (`receive.maxInputSize`, `RLIMIT_FSIZE`); a hard guarantee needs filesystem quotas or the container's cgroup limits. + The child runs with `RLIMIT_CPU` (900 s), `RLIMIT_FSIZE`, `RLIMIT_CORE=0` and, on Linux, `RLIMIT_DATA` + (`--git-memory-limit`, 8 GiB per child, so the host-wide bound is `--max-transports` times that; anonymous memory, + which stops a pack declaring a huge compressed object from growing one child without limit; + not enforced on macOS and untested in CI: a cgroup limit on the service is the real bound), a wall-clock limit + (`--git-timeout`) and directional stall deadlines (`--git-idle-timeout`: no request bytes while git waits for them, or + no room in the response because the client does not read), all of which end the transport + (`slots_are_bounded_then_released_when_the_client_disconnects`, `a_stalled_upload_is_cut_off_by_the_idle_deadline`, + `the_wall_clock_limit_ends_a_transport`, `a_push_over_the_size_cap_fails_and_changes_nothing`, + `a_full_disk_reserve_refuses_pushes_with_507_but_not_reads`). Remaining exposure: any authenticated user with read + access can hold transport slots, for `--git-timeout` (30 minutes) each by trickling a byte just inside the idle + deadline, and a user who can create repositories gets 16 slots per repository, so one account can fill the 64 global + slots and make everyone else wait out `--admission-wait` (503). There are no per-user quotas yet (F5; a per-user + limit would also throttle Engine, whose clones come from one service account, so it needs a deliberate budget). + A push's growth check credits its own written bytes back, but other pushes in flight are still counted twice (free space has dropped and they still hold the bytes), so it + errs on the side of refusing. A killed push gets SIGTERM first (git removes its lock files and quarantine directory + from its signal handlers; we wait up to one second for the leader to exit and stderr to close, not tested) and then SIGKILL; a server killed outright can still leave a stale + `refs/heads/.lock` or `objects/incoming-*` behind (cleanup is F5). +- **Process control.** One supervisor thread owns each child and its process group and signals the group while the + leader is still an unreaped zombie (so a reused group id cannot be hit); the registry holds a cancel flag, never a + group id. A client that hangs up ends the child (`a_large_push_and_clone_stream_and_a_vanished_client_ends_the_child`); + deleting a repository first closes its registry slot, cancels and waits for every transport on it, and refuses to + rename the directory while one survives (`deleting_a_repository_ends_its_running_transports_first`); shutdown + cancels what is left after the drain deadline and refuses to register anything new, so a request still queued for a slot + cannot start git afterwards (`shutdown_ends_a_running_transport`, + `a_request_queued_for_a_slot_does_not_start_git_after_shutdown`). A server killed outright + (SIGKILL, OOM) leaves children running until the container's PID namespace or the service manager's cgroup ends them. +- **Response integrity.** The CGI header block is parsed and only a whitelist of headers (content type, cache and + validator headers) is forwarded, each validated against control characters; a child that fails after the response head + was sent ends the stream with an error so the connection drops instead of finishing a truncated body with a clean + terminator: the response body ends only with an explicit end marker from the supervisor's clean verdict, so a closed + channel is an error (`a_stalled_upload_is_cut_off_by_the_idle_deadline`). Behind a proxy that speaks HTTP/1.0 to its + upstream (nginx's default) a cut stream can look like a clean end; git's own pkt-line framing still detects most + truncations, and the deployment should set `proxy_http_version 1.1`. Because `git http-backend` writes its response head + before it reads the request, an aborted push is visible to the client as a broken connection, not as a 408/413 status. +- **Transport security.** The server speaks plaintext HTTP and refuses a non-loopback bind unless told that a TLS proxy + fronts it (`--behind-proxy`) or plaintext is wanted (`--allow-plaintext`); tokens cross this socket. No TLS and no SSH + in sc-forge itself. + ## Reporting Security reports go to **toni@git-agentic.com** or GitHub private vulnerability diff --git a/docs/superpowers/plans/2026-10-23-p38-f2b-git-transport.md b/docs/superpowers/plans/2026-10-23-p38-f2b-git-transport.md new file mode 100644 index 00000000..93c1ddba --- /dev/null +++ b/docs/superpowers/plans/2026-10-23-p38-f2b-git-transport.md @@ -0,0 +1,233 @@ +# P38 F2b: smart-HTTP git transport, hooks, ref policy, admission control (plan) + +Second half of slice F2 (F2a shipped the repository API, lifecycle and git runner, #301). Spec: +`docs/superpowers/specs/2026-10-22-p38-forge-design.md` ("Git transport", "Server-side Git +operations" bullets on environment, ref namespace, protection linearization, admission control, +process ownership). The threat-model pass #1 (auth, tenancy, transport, resources) merges with this +slice as a `docs/THREAT-MODEL.md` section plus the tests it names. + +## Scope + +Routes (outside `/api/v1`, every one authenticated and authorized by the server, then handed to +`git http-backend` as a CGI): + +| Request | Needs | Notes | +|---|---|---| +| `GET /{o}/{r}.git/info/refs?service=git-upload-pack` | read | smart advertisement, protocol v0 and v2 | +| `GET /{o}/{r}.git/info/refs?service=git-receive-pack` | write | `REMOTE_USER` set only after the write check | +| `POST /{o}/{r}.git/git-upload-pack` | read | clone, fetch, fetch-by-sha, `blob:none`, shallow; gzip request bodies | +| `POST /{o}/{r}.git/git-receive-pack` | write | push; streamed; capped | +| `GET .../HEAD`, `objects/info/packs`, `objects/xx/<38 hex>`, `objects/pack/pack-<40 hex>.{pack,idx}` | read | dumb reads the recordings exercise (V#14-15, J#63) | + +Anything else under `/{o}/{r}.git/` is 404; a write method other than `git-receive-pack` is 405. +The sub-path handed to the CGI is rebuilt from this whitelist, never the client's string. +Out of F2b: PR retention refs and merge (F4), branch protection rules (F4; the policy function +is the one hook point they plug into), contents/commits API (F3), per-owner quotas (default off per +spec; the admission code leaves the seam), SSH, TLS. + +## Credentials and answers (recorded Gitea texts, sc-forge realm) + +Tokens only, never a password: `Authorization: token T`, `Bearer T`, Basic with the token as the +password under any username (incl. `oauth2`), or the token as the username with an empty password +(J#0-26). Order: authenticate, scope, repository, permission. + +| Situation | Status, body (text/plain;charset=utf-8) | +|---|---| +| no credentials | 401 `Unauthorized\n` + `WWW-Authenticate: Basic realm="sc-forge"` (+ `Cache-Control: no-store`) | +| bad / unknown / expired token, or a password | 401 `Verify\n` | +| scope missing (`read:repository` / `write:repository`), or `public-only` | 403 `Forbidden\n` | +| repository missing, not live, or no access (incl. receive-pack on a missing repo) | 404 `Repository not found` | +| read access only, receive-pack | 403 `Forbidden\n` (departure: Gitea advertises and then declines in pre-receive, recorded C#30) | +| busy (admission), disk reserve | 503 / 507, `Retry-After` on 503 | + +Departures (new in `departures.json`): realm text and `cache-control` on the 401s (J#27/31, V#1/3/20/21, +K#3/4), read-only collaborator push (C#30), and the already-present K#29/39 (404 for receive-pack on a +missing repo). Content types, no-cache headers, advertisement preambles and capability lists come from +git itself and are compared as recorded (V notes); `allow-tip-sha1-in-want allow-reachable-sha1-in-want` +follow from `uploadpack.allowAnySHA1InWant=true`. + +## Execution model + +```text +server::handle ── path matches /{o}/{r}.git/... ──► transport::handle (async) + 1 authenticate+resolve spawn_blocking: auth, scope, repos::resolve, whitelist -> Plan | Resp + 2 admit global + per-repo transport slots (tokio semaphores, deadline -> 503); + receive-pack: free-space RESERVATION min(Content-Length, --max-pack-size) -> 507 + 3 register Lifecycle registry (repo id -> live children); refuses if the repo is closing + 4 spawn std::process, own process group, env scrubbed, CGI vars, rlimits + 5 stream request body -> bounded channel -> stdin thread (cap, idle timeout) + CGI stdout -> header block parse -> bounded channel -> response body + stderr drained (capped) and logged + 6 supervise waitid(WNOWAIT) exit detect, deadline, client disconnect, shutdown: + killpg while the leader is unreaped, then reap; unregister; release slots/reservation +``` + +`std::process` plus threads (not `tokio::process`) so the proven waitid/killpg ordering from +`gitproc.rs` is reused: the group is signalled while the leader is still an unreaped zombie, so its +group id cannot be reused. Threads per transport are bounded by the admission cap. + +CGI environment is built from scratch (`env_clear`), reusing `Git::env` plus: `GIT_PROJECT_ROOT=`, +`GIT_HTTP_EXPORT_ALL=1` (only after authentication), `PATH_INFO`, `REQUEST_METHOD`, `QUERY_STRING` +(`service=...` only), `CONTENT_TYPE` (whitelisted types), `CONTENT_LENGTH` when known (else EOF-delimited), +`HTTP_CONTENT_ENCODING` (`gzip` only), `HTTP_GIT_PROTOCOL` only when it matches +`^version=[0-9]+(:[A-Za-z0-9=_.-]*)?$` (length-capped), `REMOTE_USER` (login) only after the write check, +`REMOTE_ADDR`, and for hooks `FORGE_REPO_ID`, `FORGE_USER_ID`, `FORGE_DB`. Per-spawn config adds +`transfer.hideRefs=refs/forge/`, `uploadpack.allowFilter=true`, `uploadpack.allowAnySHA1InWant=true`, +`uploadpackfilter.allow=false` with `uploadpackfilter.blob:none.allow=true`, `receive.maxInputSize=`, +`receive.autogc=false`, `receive.fsyncObjects=true`. Resource limits in `pre_exec`: `RLIMIT_CPU`, +`RLIMIT_FSIZE`, `RLIMIT_CORE=0`. + +## Hooks and ref policy + +`/hooks/pre-receive` is a **symlink to the `sc-forge` binary** (no shell, no quoting; `main` dispatches on +argv[0] being `pre-receive`), re-pointed at every start (the binary path can change; the e2e harness passes the real +binary path, never the test executable); `core.hooksPath` is pinned on every spawn, repository-local hooks +do not exist (`--template=`). `sc-forge hook pre-receive` reads `old new ref` lines, opens the database +read-only, and refuses (exit 1, one `refusing ...` line per ref on stderr, which git shows as `remote: ...`): + +- any ref outside `refs/heads/*` and `refs/tags/*`: `refusing to update : only refs/heads/* and refs/tags/* are writable` + (S#0-11; `refs/replace/*`, `refs/forge/*`, `refs/pull/*`, `refs/notes/*`, `refs/changes/*`, anything else); +- deleting the repository's default branch (J#62, recorded `pre-receive hook declined`); +- anything when the repository row is not `live` (generation check: the id names the directory, so a + recreated name cannot be confused; fails closed on a database error). + +Authorization is evaluated once at the start of the request (a revocation does not cancel an in-flight push); +branch protection is evaluated at the ref update, i.e. in this hook, through one policy function that F4 +extends (no rules exist yet). There is **no post-receive hook**: the server stamps `updated_unix` itself after receive-pack exits 0. The hook opens the +database through a purpose-built read-only opener (`db::open_readonly`: no create, `query_only`, refuses unless +`user_version == SCHEMA_VERSION`, never takes `/.lock`); a failure to open or read refuses the push (fail closed). + +## Process ownership, delete, shutdown + +A registry in `Lifecycle` maps repository id -> live transports (pgid, kill handle). `delete` first sets +`deleting` (new transports are refused), closes the id in the registry, kills the group of every registered +transport and waits (bounded) for them to unregister, then renames the directory. A transport that registers +after the id was closed is refused. SIGTERM/shutdown stops accepting, drains, then kills stragglers (the +existing drain deadline), then closes the database. Client disconnect kills the group. + +## Admission control and limits (flags) + +`--max-transports` (default 32), `--max-transports-per-repo` (8), `--git-timeout` (default 1800 s wall clock), +`--git-idle-timeout` (120 s without bytes in either direction), `--max-pack-size` (receive: default 2 GiB, +also `receive.maxInputSize`), `--max-request-bytes` for upload-pack bodies, compressed and inflated (default 16 MiB), `--min-free-bytes` +(default 1 GiB reserve), CPU limit seconds (a fixed 900 s CPU limit (a constant, not a flag)). Admission happens before any process: +slot acquisition has a deadline (503 + `Retry-After: 5`); the disk reservation is released when the push ends. +The inflated size of a gzip request body is bounded by `RLIMIT_FSIZE`/memory limits of the child plus the +timeouts; the container's cgroup limits (F5) are the hard bound and the threat model says so. + +## Threat model pass #1 (written with the code) + +`docs/THREAT-MODEL.md` gains "sc-forge: authentication, tenancy, transport, resources", each item pointing at a +test: credential classes (F1 tests), existence oracles (foreign token == missing repo, git and API alike), +tenant isolation (a repository id directory is the only path a transport can name; whitelist; `--git-dir` via +GIT_PROJECT_ROOT never discovery), ref-namespace attacks (`refs/replace/*` push refused, `GIT_NO_REPLACE_OBJECTS`, +`refs/forge/` hidden on both services), env/config injection (scrubbed env, pinned hooksPath, no repo config), +header/query/protocol injection (whitelists), resource exhaustion (admission, rlimits, timeouts, reservation, +slow clients, gzip), process control (groups, pgid reuse, disconnect, delete, shutdown), cross-tenant reads +(deleted-but-unpruned objects are readable by anyone with read access to that repository, by `allowAnySHA1InWant`: +stated, not hidden). + +## Tests (written first) + +1. Unit: path whitelist (traversal, encodings, dotfiles, `..`, `.git/config`, `objects/` shapes), credential + extraction, CGI header parser (`Status:`, multiple headers, bare LF, oversized/malformed), env builder (no + inheritance, protocol-version filter), hook policy (every ref class, default-branch delete, non-live row, + DB error fails closed), registry (close-then-register refused, kill and unregister), reservation arithmetic. +2. Real `git` e2e over a socket (new `tests/e2e_git.rs`): clone, push (new branch, ff, forced with lease, + stale lease), delete branch/tag by push, tags, fetch by sha (v0 and v2) incl. a deleted branch tip, `blob:none`, + `--depth=1` + unshallow, gzip request bodies (`http.postBuffer` small / raw POST), all credential forms (J#0-26), + anonymous and bad token texts, no access vs missing vs read-only, scope-limited token, `refs/replace/*` and + `refs/pull/*` pushes refused with the recorded message, default-branch deletion refused, hidden `refs/forge/*` + never advertised, advertisement bytes and headers (V), a push that exceeds `--max-pack-size`, 507 on a full + reserve, 503 when slots are exhausted, client disconnect kills the group (no stray `git` processes), timeout, + delete while a clone streams (child killed, no leftover directory), concurrent pushes to one ref (one wins), + a hook failure leaves refs unchanged. +3. Conformance: `check` compares the recorded git command for git steps (#297); the gate adds `J`, `V`, `S`, `K`, + `M`, `C` and `N` past their clones where the scenario needs no unbuilt route; the rest keep pinned stops in + `slice-pending.json`. New departures listed above. + +## Order of work + +1. departures and the gate decision (run `check` raw, see what J/V/S/K/M/C/N need), `check` git-command comparison +2. path whitelist + git credential/answer layer (no process yet) with tests +3. hook policy + `hook` subcommand + shims + tests +4. CGI spawn/stream/supervise + registry + admission + limits, wired into `server::handle` and `Lifecycle::delete` +5. e2e git tests, then threat model text, spec "as built", README, ROADMAP +6. independent review (Opus + Sol) of the final diff; fix; PR + +## Risks and open points + +- Streaming through `server::handle`: the body must not be buffered for git routes; the JSON body cap path stays + for the API. A regression test asserts a 50 MiB push streams in bounded memory. +- `git http-backend` behaviour differs across git versions (chunked/EOF-delimited request bodies, gzip handling, + `Status:` forms). CI runs the runners' git; the minimum is 2.40. The e2e suite is the guard. +- macOS vs Linux: `waitid`, rlimits (`RLIMIT_FSIZE` semantics), `statvfs` field widths. +- The hook runs a second `sc-forge` process per push; its startup cost is paid on every push (one SQLite open). +- Disk reservation is accounting, not enforcement: a push can still exceed its reservation (bounded by + `RLIMIT_FSIZE` and `receive.maxInputSize`); a hard guarantee needs filesystem quotas (F5). + +## Decisions after independent review (Opus and Sol; these supersede any conflicting text above) + +1. **Gzip.** `git http-backend` buffers an upload-pack request in RAM (`GIT_HTTP_MAX_REQUEST_BUFFER`, default + 10 MiB) and its inflate path has no expanded-byte counter, and `RLIMIT_FSIZE` does not bound pipes or heap. So + the **server inflates gzip request bodies itself** with a hard cap on *inflated* bytes (`--max-request-bytes`), + streams the plain bytes to the CGI and never passes `Content-Encoding`; the CGI also gets + `GIT_HTTP_MAX_REQUEST_BUFFER` set from the flag. Gzip on `git-receive-pack` is 415 (git never gzips pushes). +2. **No post-receive; read-only hook DB** (above). The hook is a symlink; paths need no quoting; hook compares the + default branch **case-insensitively** (as F2a's `delete_branch`) and detects a delete by an all-zero new id of any + length; tests cover mixed-case deletion in atomic and ordinary batches with every ref asserted unchanged. +3. **Process ownership.** Each transport has exactly one supervisor thread that owns the child and signals its group + while the leader is still an unreaped zombie (the F2a `waitid(WNOWAIT)` ordering). The registry holds a + **cancel handle** (an atomic flag the supervisor polls, backoff 2 ms to 50 ms), never a pgid, so no one can signal a + reused group. Registration, spawn and cancellation are serialized per transport; a cancel requested before spawn + prevents the spawn. +4. **Registry vs delete.** Registry: `Mutex>`. `delete` (already under the + per-repo write lock) sets `deleting`, `close(id)`, cancels every child, waits (bounded 10 s) for them to + unregister, and **refuses to rename while any survives** (reopens the slot and reports an error). `register` + refuses a closed slot and re-reads the row's state under the registry mutex. Transports do **not** hold the shared + `with_read` lock (a 30-minute push would block default-branch changes and branch deletes). A push that deletes a + branch while a PATCH makes it the default can leave `HEAD` dangling (`empty: true`); the row stays authoritative. +5. **Shutdown.** `serve` cancels every registered transport and reaps it before returning; a server killed outright + (SIGKILL, OOM) leaves children until the next start or the container's PID namespace / systemd cgroup ends them + (stated in the threat model). +6. **Backpressure and slow clients.** Directional progress deadlines: no request bytes for `--git-idle-timeout` while + the child waits for input, or no response progress (the channel to the client stays full) for the same time, cancels + the transport. Pumps are `std::thread`s using `try_send` loops that also watch the cancel flag, so a stalled reader + cannot pin a thread past cancellation. Thread count is bounded by admission. +7. **Admission, defined.** Order: global transport slot, then per-repo slot, one `--admission-wait` deadline for both + (default 30 s, so an agent cloning one repo many times queues instead of failing at the 9th clone; limits 64 global, + 16 per repo), then (receive-pack only) the disk reservation; slot exhaustion is 503 + `Retry-After`, reserve + exhaustion is 507. The F2a API git gate is separate (documented). git sends pushes larger than `http.postBuffer` as + chunked, so most pushes have no `Content-Length`: reservation starts at `--push-reserve` (default 256 MiB) and grows + in 64 MiB steps as bytes are forwarded up to `--max-pack-size`; a step that cannot be reserved aborts the push + (logged; the client sees a dropped connection, since the response cannot be rewritten after the upload started). + The reservation is accounting, not enforcement (a hard guarantee needs filesystem quotas, F5). +8. **Config.** `receive.advertisePushOptions=true` (recorded V#12); push options are accepted and ignored (never + exposed to the hook as authority). `GIT_HTTP_MAX_REQUEST_BUFFER` as above. +9. **Recorded behaviours that need departures** (added with the code): v2 advertisement has no `# service=` preamble in + stock git (V#5 note, J#35 body); Gitea's private `cache-control` on POST results and 401s (V#16/18/1/3/20/21); + realm text; read-only collaborator pushes (C#30, S#13) are 403 at the advertisement; **HEAD never moves on a push + and an unborn default stays unborn after a first push to another branch** (N#15-16, N#24-28 notes: the row is the + authority and the first-push heuristic of Gitea is not copied). +10. **Gate mechanics.** `--pending` only tolerates a SetupError at the pinned stop; recorded calls to unbuilt routes + (J#56-57 `/tags`, K `/contents`, N#6 `/contents`) cannot be hidden. The pending file gains **`defer`** entries per + scenario (`{"J": {"defer": [[56, "GET /api/v1/repos/acme/smart/tags"], ...]}}`): the named exchanges are executed + but not compared (reported as deferred, identity pinned by method and path, a changed recorded call fails the + run); an exchange that starts passing is reported so the entry can be removed. Supported scenarios and prefixes are + decided from a raw run before coding the gate and listed in the spec; S needs an unrecorded PR creation before + exchange 0 (F4) and stays out. +11. **`check` git-command comparison (#297)** is independent tooling and ships as its own small PR first. +12. **Spec drift and exceptions.** `FORGE_GENERATION` is replaced by the immutable id (AUTOINCREMENT, never reused); + the spec's "as built" says so. The transport sits beside the router table: it has its own declared table + (`transport::ROUTES`, class/scope/permission per sub-path) enumerated by a test, an explicit, recorded exception to + "every route in `routes::ROUTES`". Only `/{o}/{r}.git/...` is served (Gitea also serves it without `.git`; Engine's + recorded clients use `.git`). Dumb `info/refs` without `service=` is pinned 404 (git's dumb listing ignores + `hideRefs`). +13. **Test mechanics.** No fixed sleeps: readiness barriers and bounded polling; "no stray git" asserts + `kill(-pgid, 0) == ESRCH`; memory-boundedness is asserted by blocking the child's stdin and checking the sender + blocks (channel bound), not RSS; real binary (not the test harness) for hook tests; explicit gzip, chunked and + EOF-delimited request cases; run on Linux and macOS CI with git >= 2.40. +14. **Order of work (revised).** 0) `check` git-command comparison as its own PR; 1) departures + gate from a raw run; + 2) whitelist + git credential/answer layer; 3) hook + opener + symlink; 4a) minimal CGI spawn and streaming with a + real-git clone/push e2e before anything else is built on it; 4b) registry, delete integration, shutdown; 4c) + admission, reservation, limits, rlimits; 5) full e2e, threat model, spec, README; 6) reviews; PR. diff --git a/docs/superpowers/specs/2026-10-22-p38-forge-design.md b/docs/superpowers/specs/2026-10-22-p38-forge-design.md index 3b07e780..dcada1c6 100644 --- a/docs/superpowers/specs/2026-10-22-p38-forge-design.md +++ b/docs/superpowers/specs/2026-10-22-p38-forge-design.md @@ -753,10 +753,65 @@ the smart-HTTP transport, hooks, ref policy, admission control and threat-model every repository directory to `orphans/` (safe, and recoverable by restoring the database). Not verified against Gitea's source: `/user/repos` is scoped `write:user`/`read:user` as recorded (B#26, B#34); whether Gitea also requires a repository scope there is open. -- **Conformance gate.** `forge-check.sh` replays `A`, `B`, `C`, `L`, `N` and `R`; `B` stops at - its first contents call (F3), `C` and `N` at their first `git clone` (F2b), and `X`, `D`, - `K`, `M` stay out until their routes exist (they make recorded calls to unbuilt routes, which - `--pending` cannot hide). 97 recorded exchanges are verified for real. +- **Conformance gate.** (F2a; superseded by the F2b gate below.) `forge-check.sh` replayed `A`, `B`, `C`, `L`, `N` and `R`, + with `C` and `N` stopping at their first `git clone`. + +### F2b decisions (as built) + +F2b is the git half of F2 (plan: `docs/superpowers/plans/2026-10-23-p38-f2b-git-transport.md`, with its +"Decisions after independent review"). Differences from the plan are listed last. + +- **Routes.** `/{o}/{r}.git/{info/refs, git-upload-pack, git-receive-pack, HEAD, objects/info/packs, objects/xx/<38 hex>, + objects/pack/pack-.{pack,idx}}` only, served by `git http-backend` run as a CGI; everything else under `.git/` is + 404, a wrong method 405, a dumb `info/refs` (no `service=`) 404 (git's dumb listing ignores `hideRefs`). The transport + has its own whitelist (`transport::parse`) beside `routes::ROUTES`: an explicit, recorded exception to "every route is in + the table". Gitea also serves the path without `.git`; sc-forge does not. +- **Order of checks** on a git request: credentials (401), method (405), scope (403), repository and existence (404), + permission (403), then encoding (415), declared body size (413), admission (503), gzip inflation (400/413), disk + reservation (507), process start. + Answers are the recorded Gitea texts; the realm is `Gitea`, and 401s and RPC results carry Gitea's + `Cache-Control: max-age=0, private, must-revalidate, no-transform`. +- **Process model.** `server::handle` routes git paths before the bounded body read. A transport is a child + (`std::process`, own group) with a supervisor thread (exit detection by `waitid(WNOWAIT)`, `killpg` before reaping, + deadline, cancel flag), a stdin writer, a stdout reader that parses the CGI head then streams, and a stderr tail that is + logged when the child fails. A gzip request body is inflated whole before the spawn (cap on compressed and inflated + bytes); other bodies go through a bounded channel fed by an async pump (size cap, idle deadline, reservation + growth). The response body is a channel read by hyper that ends only on an explicit marker from a clean exit, and + dropping it cancels the transport. A transport we end gets SIGTERM, then SIGKILL after 1 s. `Lifecycle::registry` maps repository id to cancel + handles; `delete` closes the slot, cancels, waits up to 10 s and refuses to rename while a transport survives. +- **Hook.** `pre-receive` is a symlink `/hooks/pre-receive` to the `sc-forge` binary (argv[0] dispatch, no shell); + there is no post-receive hook, the server stamps `updated_unix` itself after `receive-pack` exits 0. + `core.hooksPath` is pinned per spawn. +- **Ref policy** (`policy.rs`, shared by the hook and F2a's branch deletes): only `refs/heads/*` and `refs/tags/*` are + writable; the default branch cannot be deleted (Unicode lower case compare); a repository that is not `live` refuses + everything. Push options are advertised and ignored. HEAD never moves on a push: the row is the authority, an unborn + default stays unborn (departures N#13/16/22/26). +- **Known race.** The hook reads the default branch without the per-repository lock (a push can run for 30 minutes), so a + `PATCH default_branch` racing a push that deletes the new default can leave `HEAD` and the row naming a branch that + no longer exists (`empty: true` until the next push or PATCH). The row stays the authority; closing it needs a + lease between receive-pack and the PATCH, which F4's branch protection will need anyway. +- **Flags.** `--max-transports` (64), `--max-transports-per-repo` (16), `--admission-wait` (30 s), `--git-timeout` + (1800 s), `--git-idle-timeout` (120 s), `--max-pack-size` (2 GiB), `--max-request-bytes` (16 MiB), + `--min-free-bytes` (1 GiB), `--push-reserve` (256 MiB), `--git-memory-limit` (8 GiB, Linux). CPU limit 900 s is a constant. The F2a runner gate + (`--git-concurrency`) is separate and governs the API's own git calls. +- **Commit authors.** A branch's `commit.author.username`/`committer.username` is the login of the one user whose email + matches the commit's address, empty if none or ambiguous (recorded K#32, N#5). +- **Departures** (all in `departures.json` with reasons): read-only collaborator pushes are 403 at the advertisement + (C#30, K#34; Gitea declines in pre-receive), the default-branch delete message (J#62), protocol v2 advertisements have no + `# service=` preamble (V#5), the unborn-default behaviour above (N). Gitea's behaviour that the tests cannot reproduce + and sc-forge does not copy: moving HEAD to the first pushed branch. +- **Conformance gate.** `forge-check.sh` replays `A B C J K L M N R V` (10 scenarios). `slice-pending.json` now has + `{"stop": [N, text], "defer": [[index, "METHOD /path"], ...]}` entries: a deferred exchange still runs and must be the + recorded call, but its mismatch waits for a later slice, and it fails the run when it starts passing. `B` stops at its + first contents call; J's `/tags` and `/git/refs/tags`, K's and N's `/contents`, and K's org listing counts (which depend + on scenarios that need F3/F4 routes) are deferred. `S` needs a PR (F4); `X`, `D`, `E`-`I`, `O`-`Q`, `T`, `U`, `W` stay out. + A departed git step still has its command line compared; credentialed URLs compare equal under `http` and `https`. +- **Differences from the plan.** Admission takes the per-repository slot first (the plan said global first, which lets a + flood on one repository hold global slots while waiting); the realm and RPC cache policy are Gitea's (the plan departed + from both); fetch-by-id of a deleted branch tip works for anyone with read access to that repository, as stated in the + threat model. Not built: a test that asserts memory is bounded by blocking the child's stdin (the bounded channels are + the mechanism; the 20 MiB push/clone test exercises the streaming path), per-owner quotas, `gc` of killed pushes' quarantine + directories (F5). ## Open questions