diff --git a/ROADMAP.md b/ROADMAP.md index 0abe399b..69a7b599 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -705,7 +705,7 @@ capture) through F6 (acceptance on the first deployment); F0, F1 (server skeleton, users, organizations, tokens, `admin` CLI), F2a (repositories API, lifecycle and reconciler, collaborators, branches) and F2b (smart-HTTP git transport via `git http-backend`, `pre-receive` ref policy, admission control, threat-model pass #1) and F3a (content reads: refs, tags, commits, -trees, contents, archives; writes are F3b) are built. Hosted repos are +trees, contents, archives) and F3b (contents writes: create, update, delete as server-made commits) are built. Hosted repos are bare Git and get none of the `.sc/` features. Other follow-up work is tracked as GitHub issues and listed under Deferred diff --git a/crates/forge/conformance/slice-pending.json b/crates/forge/conformance/slice-pending.json index e9653d31..b1698cbd 100644 --- a/crates/forge/conformance/slice-pending.json +++ b/crates/forge/conformance/slice-pending.json @@ -15,38 +15,6 @@ ] ] }, - "E": { - "stop": [ - 42, - "KeyError: 'sha'" - ], - "defer": [ - [ - 36, - "POST /api/v1/repos/acme/hist/contents/new.txt" - ], - [ - 37, - "POST /api/v1/repos/acme/hist/contents/new.txt" - ], - [ - 38, - "PUT /api/v1/repos/acme/hist/contents/README.md" - ], - [ - 39, - "PUT /api/v1/repos/acme/hist/contents/README.md" - ], - [ - 40, - "PUT /api/v1/repos/acme/hist/contents/README.md" - ], - [ - 41, - "POST /api/v1/repos/acme/hist/contents/branch-file.txt" - ] - ] - }, "K": { "defer": [ [ diff --git a/crates/forge/src/cgi.rs b/crates/forge/src/cgi.rs index 132ef65d..5cc5a5bb 100644 --- a/crates/forge/src/cgi.rs +++ b/crates/forge/src/cgi.rs @@ -105,6 +105,17 @@ impl Transports { } } + /// Reserve `need` bytes of disk for a server-made write against the same floor and pool as pushes + /// (`None`: the disk cannot promise it). + pub fn reserve_disk( + &self, + data_dir: &std::path::Path, + need: u64, + ) -> Option { + let free = crate::transport::free_bytes(data_dir).unwrap_or(0); + self.reserve.reserve(free, self.cfg.min_free, need) + } + fn repo_slots(&self, repo: i64) -> Arc { let mut m = self.per_repo.lock().unwrap_or_else(|e| e.into_inner()); if m.len() > 256 { @@ -182,6 +193,11 @@ fn buffered(r: crate::http::Resp) -> Response { .unwrap_or_else(|_| fallback(500)) } +/// The answer to a request that meets the server's shutdown. +fn shutting_down() -> crate::http::Resp { + text(503, "Service Unavailable\n").with("retry-after", "5".into()) +} + fn fallback(status: u16) -> Response { let mut r = Response::new( http_body_util::Full::new(Bytes::from_static(b"internal server error\n")) @@ -840,6 +856,8 @@ pub async fn handle(app: Arc, req: Req, body: Incoming, path: GitPath) -> R match tokio::task::spawn_blocking(move || transport::authorize(&app, &req, path)).await { Ok(Ok(p)) => p, Ok(Err(r)) => return buffered(r), + // A cancelled blocking task means the runtime is shutting down; a panic is an internal error. + Err(e) if e.is_cancelled() => return buffered(shutting_down()), Err(_) => return buffered(text(500, "internal server error\n")), } }; @@ -936,13 +954,12 @@ pub async fn handle(app: Arc, req: Req, body: Incoming, path: GitPath) -> R let started = match tokio::task::spawn_blocking(move || launch(launch_args)).await { Ok(Ok(s)) => s, Ok(Err(LaunchError::Gone)) => return buffered(transport::repo_not_found()), - Ok(Err(LaunchError::Stopped)) => { - return buffered(text(503, "Service Unavailable\n").with("retry-after", "5".into())) - } + Ok(Err(LaunchError::Stopped)) => return buffered(shutting_down()), Ok(Err(LaunchError::Io(e))) => { eprintln!("sc-forge: could not start git: {e}"); return buffered(text(500, "internal server error\n")); } + Err(e) if e.is_cancelled() => return buffered(shutting_down()), Err(_) => return buffered(text(500, "internal server error\n")), }; let Started { diff --git a/crates/forge/src/content_api.rs b/crates/forge/src/content_api.rs index cfdfb8b7..f19562fb 100644 --- a/crates/forge/src/content_api.rs +++ b/crates/forge/src/content_api.rs @@ -18,12 +18,12 @@ use serde_json::{json, Value}; use std::collections::HashMap; use std::path::PathBuf; -fn dir(ctx: &Ctx, a: &Access) -> PathBuf { +pub(crate) fn dir(ctx: &Ctx, a: &Access) -> PathBuf { ctx.app.life.repo_dir(a.repo.id) } /// `{base}/api/v1/repos/{owner}/{repo}`. -fn api_base(ctx: &Ctx, a: &Access) -> String { +pub(crate) fn api_base(ctx: &Ctx, a: &Access) -> String { format!( "{}/api/v1/repos/{}/{}", ctx.app.base, a.repo.owner, a.repo.name @@ -31,7 +31,7 @@ fn api_base(ctx: &Ctx, a: &Access) -> String { } /// `{base}/{owner}/{repo}`: the web URLs Gitea puts in `html_url`. -fn web_base(ctx: &Ctx, a: &Access) -> String { +pub(crate) fn web_base(ctx: &Ctx, a: &Access) -> String { format!("{}/{}/{}", ctx.app.base, a.repo.owner, a.repo.name) } @@ -55,7 +55,7 @@ fn flag(ctx: &Ctx, k: &str, default: bool) -> bool { } /// A git failure that is the request's doing, as opposed to the repository's. -fn read_err(ctx: &Ctx, a: &Access, e: GitError) -> Resp { +pub(crate) fn read_err(ctx: &Ctx, a: &Access, e: GitError) -> Resp { match e { GitError::OutputTooLarge => error(&ctx.app.base, 413, "the result is too large to serve"), GitError::Timeout { .. } => error(&ctx.app.base, 504, "git timed out"), @@ -123,7 +123,7 @@ impl<'a> Users<'a> { } } -fn verification() -> Value { +pub(crate) fn verification() -> Value { json!({"verified": false, "reason": "gpg.error.not_signed_commit", "signature": "", "signer": null, "payload": ""}) } @@ -588,33 +588,12 @@ fn content_json( exact_last: bool, ) -> Result { let (git, d) = (&ctx.app.life.git, dir(ctx, a)); - let api = api_base(ctx, a); - let web = web_base(ctx, a); - let enc_path = e - .path - .split('/') - .map(pct_encode) - .collect::>() - .join("/"); - let enc_ref = refname - .split('/') - .map(pct_encode) - .collect::>() - .join("/"); - let query_ref = pct_encode(refname); let last = if exact_last { git_err(ctx, a, gitobj::last_commit(git, &d, commit, &e.path))?.unwrap_or_default() } else { commit.to_string() }; - let name = e.path.rsplit('/').next().unwrap_or(&e.path); - let seg = kind.segment(); - let ty = match (e.kind.as_str(), e.mode.as_str()) { - ("tree", _) => "dir", - ("commit", _) => "submodule", - (_, "120000") => "symlink", - _ => "file", - }; + let ty = entry_type(e); let size = e.size.unwrap_or(0); let (mut encoding, mut content, mut target) = (Value::Null, Value::Null, Value::Null); // A link target is a path (PATH_MAX); a bigger blob is not read, and a listing reads at most @@ -627,11 +606,57 @@ fn content_json( encoding = json!("base64"); content = Value::String(b64::encode(&b)); } + Ok(content_value( + ctx, a, refname, kind, e, &last, encoding, content, target, + )) +} + +/// The type string of a tree entry in a contents object. +fn entry_type(e: &TreeEntry) -> &'static str { + match (e.kind.as_str(), e.mode.as_str()) { + ("tree", _) => "dir", + ("commit", _) => "submodule", + (_, "120000") => "symlink", + _ => "file", + } +} + +/// The contents object of `e` as of ref `refname` (also what a contents write answers with). +#[allow(clippy::too_many_arguments)] +pub(crate) fn content_value( + ctx: &Ctx, + a: &Access, + refname: &str, + kind: RefKind, + e: &TreeEntry, + last: &str, + encoding: Value, + content: Value, + target: Value, +) -> Value { + let api = api_base(ctx, a); + let web = web_base(ctx, a); + let enc_path = e + .path + .split('/') + .map(pct_encode) + .collect::>() + .join("/"); + let enc_ref = refname + .split('/') + .map(pct_encode) + .collect::>() + .join("/"); + let query_ref = pct_encode(refname); + let name = e.path.rsplit('/').next().unwrap_or(&e.path); + let seg = kind.segment(); + let ty = entry_type(e); + let size = e.size.unwrap_or(0); let self_url = format!("{api}/contents/{enc_path}?ref={query_ref}"); // Recorded: directories point at `git/blobs/` too. let git_url = format!("{api}/git/blobs/{}", e.oid); let html = format!("{web}/src/{seg}/{enc_ref}/{enc_path}"); - Ok(json!({ + json!({ "name": name, "path": e.path, "sha": e.oid, @@ -647,7 +672,7 @@ fn content_json( "download_url": if ty == "file" { json!(format!("{web}/raw/{seg}/{enc_ref}/{enc_path}")) } else { Value::Null }, "submodule_git_url": null, "_links": {"self": self_url, "git": git_url, "html": html}, - })) + }) } /// Entries of one listing whose `last_commit_sha` is computed exactly. diff --git a/crates/forge/src/content_write.rs b/crates/forge/src/content_write.rs new file mode 100644 index 00000000..7160bce5 --- /dev/null +++ b/crates/forge/src/content_write.rs @@ -0,0 +1,540 @@ +//! `POST`/`PUT`/`DELETE /repos/{o}/{r}/contents/{path..}` (F3b): one file changed by a server-made +//! commit. The commit is built from object ids with plumbing (`gitwrite`) and the ref moves last, by +//! compare-and-swap; the response is built before that, so nothing is read from git afterwards. +//! Plan: `docs/superpowers/plans/2026-10-24-p38-f3b-contents-writes.md`. + +use crate::api::body; +use crate::b64; +use crate::content_api::{api_base, content_value, dir, read_err, verification, web_base}; +use crate::gitobj::{RefKind, TreeEntry}; +use crate::gitproc::Identity; +use crate::gitwrite::{self as gw, Op, Outcome, Refuse, WriteErr}; +use crate::http::{error, not_found, Resp}; +use crate::lifecycle::WriteRun; +use crate::policy; +use crate::refname; +use crate::repo_api::acc; +use crate::repos::{self, Access, Repo}; +use crate::routes::Ctx; +use crate::timefmt::{now, rfc3339}; +use serde_json::{json, Value}; +use std::time::{Duration, Instant}; + +/// Publication attempts before the write answers 409. +const ATTEMPTS: usize = 3; +/// How long the whole operation may take, checked between attempts and between the walk, the rewrite and the commit. +const DEADLINE: Duration = Duration::from_secs(60); +/// How long a write waits for the repository's write gate. +const GATE_WAIT: Duration = Duration::from_secs(10); +/// Pause before rebuilding after a ref lock was held. +const BACKOFF: Duration = Duration::from_millis(50); +/// Disk promised beyond the content itself, up front; each attempt then reserves the tree bytes it read. +const RESERVE_SLACK: u64 = 8 << 20; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Verb { + Post, + Put, + Delete, +} + +/// A validated request. +struct Write { + verb: Verb, + path: String, + bytes: Vec, + message: String, + branch: Option, + new_branch: Option, + sha: Option, +} + +fn unprocessable(ctx: &Ctx, msg: &str) -> Resp { + error(&ctx.app.base, 422, msg) +} + +fn no_branch(ctx: &Ctx, name: &str) -> Resp { + not_found( + &ctx.app.base, + "The target couldn't be found.", + Some(&format!("branch does not exist [name: {name}]")), + ) +} + +/// An optional string field: absent, `null` and `""` are all "not given". +fn opt_str(ctx: &Ctx, b: &Value, key: &str) -> Result, Resp> { + match b.get(key) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(s)) if s.is_empty() => Ok(None), + Some(Value::String(s)) => Ok(Some(s.clone())), + Some(_) => Err(unprocessable(ctx, &format!("[{key}]: must be a string"))), + } +} + +fn parse(ctx: &Ctx, verb: Verb) -> Result { + let b = body(ctx)?; + let path = ctx.param("p").to_string(); + if opt_str(ctx, &b, "from_path")?.is_some_and(|f| f != path) { + return Err(unprocessable(ctx, "from_path (rename) is not supported")); + } + if !gw::write_path_ok(&path) { + return Err(unprocessable( + ctx, + &format!("invalid file path [path: {path}]"), + )); + } + let sha = opt_str(ctx, &b, "sha")?; + if verb != Verb::Post && sha.is_none() { + return Err(unprocessable(ctx, "[SHA]: Required")); + } + let bytes = if verb == Verb::Delete { + Vec::new() + } else { + let Some(c) = opt_str(ctx, &b, "content")? else { + // an empty file is legal; a missing field is not + return match b.get("content") { + Some(Value::String(_)) => Ok(Write { + verb, + bytes: Vec::new(), + ..parse_rest(ctx, verb, &b, path, sha)? + }), + _ => Err(unprocessable(ctx, "[Content]: Required")), + }; + }; + b64::decode(&c).ok_or_else(|| unprocessable(ctx, "invalid base64 content"))? + }; + Ok(Write { + bytes, + ..parse_rest(ctx, verb, &b, path, sha)? + }) +} + +/// Message and branch fields (the part of the request every verb shares). +fn parse_rest( + ctx: &Ctx, + verb: Verb, + b: &Value, + path: String, + sha: Option, +) -> Result { + let verbword = match verb { + Verb::Post => "Add", + Verb::Put => "Update", + Verb::Delete => "Delete", + }; + let message = opt_str(ctx, b, "message")?.unwrap_or_else(|| format!("{verbword} {path}")); + if message.contains('\0') { + return Err(unprocessable(ctx, "[Message]: must not contain NUL")); + } + let message = format!("{}\n", message.trim_end()); + let name = |k: &str| -> Result, Resp> { + match opt_str(ctx, b, k)? { + Some(n) if !refname::valid_branch(&n) => { + Err(unprocessable(ctx, &format!("[{k}]: invalid branch name"))) + } + other => Ok(other), + } + }; + let branch = name("branch")?; + let new_branch = name("new_branch")?.filter(|n| Some(n) != branch.as_ref()); + Ok(Write { + verb, + path, + bytes: Vec::new(), + message, + branch, + new_branch, + sha, + }) +} + +/// The `POST`, `PUT` and `DELETE` handler. +pub fn write_contents(ctx: &mut Ctx) -> Resp { + let a = match acc(ctx) { + Ok(a) => a.clone(), + Err(r) => return r, + }; + let verb = match ctx.req.method.as_str() { + "POST" => Verb::Post, + "PUT" => Verb::Put, + "DELETE" => Verb::Delete, + _ => return error(&ctx.app.base, 405, "method not allowed"), + }; + let w = match parse(ctx, verb) { + Ok(w) => w, + Err(r) => return r, + }; + if !gw::content_ok(&w.path, &w.bytes) { + return unprocessable(ctx, "content is not allowed at this path"); + } + let ctx: &Ctx = ctx; + match ctx + .app + .life + .with_write_row(ctx.conn, a.repo.id, GATE_WAIT, |row| run(ctx, &a, row, &w)) + { + Ok(WriteRun::Ran(r)) => r, + Ok(WriteRun::Busy) => error( + &ctx.app.base, + 503, + "this repository is being written to, try again", + ), + Ok(WriteRun::Gone) => not_found(&ctx.app.base, "The target couldn't be found.", None), + Err(e) => ctx.app.internal(&e), + } +} + +/// The address git will record: surrounding blanks are stripped by git, so strip them here too and the +/// response matches the commit. +fn commit_email(user: &crate::store::User) -> String { + let e = user.email.trim(); + if e.is_empty() { + format!("{}@noreply.localhost", user.login) + } else { + e.to_string() + } +} + +fn refuse(ctx: &Ctx, path: &str, r: Refuse) -> Resp { + match r { + Refuse::AlreadyExists => unprocessable( + ctx, + &format!("repository file already exists [path: {path}]"), + ), + Refuse::Blocked(p) => { + unprocessable(ctx, &format!("repository file already exists [path: {p}]")) + } + Refuse::NoFile => not_found( + &ctx.app.base, + "The target couldn't be found.", + Some(&format!("repository file does not exist [path: {path}]")), + ), + Refuse::NotRegular => { + unprocessable(ctx, &format!("path is not a regular file [path: {path}]")) + } + Refuse::ShaMismatch { given, expected } => unprocessable( + ctx, + &format!("sha does not match [given: {given}, expected: {expected}]"), + ), + } +} + +fn denied(ctx: &Ctx) -> Resp { + error( + &ctx.app.base, + 403, + &format!("user cannot commit to repo [user: {}]", ctx.who.user.login), + ) +} + +/// A branch name that equals, contains or lies under an existing branch name. +fn collides(existing: &[String], new: &str) -> Option { + for n in existing { + if n == new { + return Some(true); + } + if n.starts_with(&format!("{new}/")) || new.starts_with(&format!("{n}/")) { + return Some(false); + } + } + None +} + +/// One write, under the gate and the shared lifecycle lock, against the fresh `row`. +fn run(ctx: &Ctx, a: &Access, row: &Repo, w: &Write) -> Resp { + let started = Instant::now(); + let (git, d) = (&ctx.app.life.git, dir(ctx, a)); + let user = &ctx.who.user; + let default = row.default_branch.clone(); + let base_name = w.branch.clone().unwrap_or_else(|| default.clone()); + let new_branch = w.new_branch.clone().filter(|n| *n != base_name); + let target_name = new_branch.clone().unwrap_or_else(|| base_name.clone()); + // Gitea checks protection before it looks at the tree. + if policy::check_contents_write(&target_name, new_branch.is_some()).is_err() { + return denied(ctx); + } + let comps: Vec<&str> = w.path.split('/').collect(); + let op = match (w.verb, &w.sha) { + (Verb::Post, _) => Op::Create, + (Verb::Put, Some(s)) => Op::Update(s.clone()), + (Verb::Delete, Some(s)) => Op::Delete(s.clone()), + _ => return unprocessable(ctx, "[SHA]: Required"), + }; + let email = commit_email(user); + let when = now(); + let Some(identity) = Identity::new(&user.login, &email).map(|i| i.at(when)) else { + return ctx.app.internal(&"unusable commit identity"); + }; + // Disk is promised only once the request has passed its preconditions (a refused one is 422, not 507). + let mut held = None; + // Written after the first successful precondition check, so a refused request stores nothing. + let mut blob: Option = None; + let base_ref = refname::branch_ref(&base_name); + let mut locked = false; + for attempt in 0..ATTEMPTS { + if started.elapsed() > DEADLINE { + return error(&ctx.app.base, 503, "the write took too long, try again"); + } + let old = match gw::read_ref(git, &d, &base_ref) { + Ok(o) => o, + Err(e) => return read_err(ctx, a, e), + }; + // An unborn default branch is where a repository's first commit goes. + let bootstrap = + old.is_none() && w.verb == Verb::Post && new_branch.is_none() && base_name == default; + if old.is_none() && !bootstrap { + return no_branch(ctx, &base_name); + } + if let Some(n) = &new_branch { + match gobj_branches(ctx, a) { + Ok(rows) => match collides(&rows, n) { + Some(true) => { + return unprocessable(ctx, &format!("branch already exists [name: {n}]")) + } + Some(false) => { + return unprocessable( + ctx, + &format!("branch name conflicts with an existing branch [name: {n}]"), + ) + } + None => {} + }, + Err(r) => return r, + } + } + let root = match &old { + Some(o) => match gw::read_ref(git, &d, &format!("{o}^{{tree}}")) { + Ok(Some(t)) => Some(t), + Ok(None) => return ctx.app.internal(&"base commit has no tree"), + Err(e) => return read_err(ctx, a, e), + }, + None => None, + }; + let mut budget = gw::Budget::new(started + DEADLINE); + let walk = match gw::walk(git, &d, root.as_deref(), &comps, &mut budget) { + Ok(w) => w, + Err(e) => return write_err(ctx, a, e), + }; + if started.elapsed() > DEADLINE { + return error(&ctx.app.base, 503, "the write took too long, try again"); + } + let prev_mode = match gw::check(&op, &comps, &walk) { + Ok(m) => m, + Err(r) => return refuse(ctx, &w.path, r), + }; + // The content and slack once, and for this attempt the tree bytes the walk read (the rewrite + // stores about as many). + if held.is_none() { + held = ctx + .app + .transports + .reserve_disk(&ctx.app.data_dir, w.bytes.len() as u64 + RESERVE_SLACK); + if held.is_none() { + return error(&ctx.app.base, 507, "insufficient storage"); + } + } + let trees = (gw::TOTAL_LISTING - budget.listing) as u64 + (1 << 20); + let _trees_held = match ctx.app.transports.reserve_disk(&ctx.app.data_dir, trees) { + Some(h) => h, + None => return error(&ctx.app.base, 507, "insufficient storage"), + }; + if blob.is_none() && w.verb != Verb::Delete { + match gw::hash_blob(git, &d, &w.bytes, started + DEADLINE) { + Ok(b) => blob = Some(b), + Err(e) => return read_err(ctx, a, e), + } + } + let mode = prev_mode.unwrap_or_else(|| "100644".to_string()); + let file = blob.as_deref().map(|b| (mode.as_str(), b)); + let tree = match gw::rebuild(git, &d, &walk, &comps, file, started + DEADLINE) { + Ok(t) => t, + Err(e) => return write_err(ctx, a, e), + }; + if started.elapsed() > DEADLINE { + return error(&ctx.app.base, 503, "the write took too long, try again"); + } + let commit = match gw::commit_tree( + git, + &d, + &tree, + old.as_deref(), + &w.message, + &identity, + started + DEADLINE, + ) { + Ok(c) => c, + Err(e) => return read_err(ctx, a, e), + }; + let creating = new_branch.is_some() || bootstrap; + let (full, expected) = if creating { + (refname::branch_ref(&target_name), gw::ZERO.to_string()) + } else { + (base_ref.clone(), old.clone().unwrap_or_default()) + }; + let resp = response( + ctx, + a, + w, + &target_name, + &commit, + &tree, + old.as_deref(), + blob.as_deref(), + &mode, + when, + ); + if started.elapsed() > DEADLINE { + return error(&ctx.app.base, 503, "the write took too long, try again"); + } + // The linearization point of branch protection: just before the ref moves. + if policy::check_contents_write(&target_name, creating).is_err() { + return denied(ctx); + } + let failure = match gw::update_ref(git, &d, &full, &commit, &expected) { + Ok(None) => return published(ctx, a, resp), + Ok(Some(stderr)) => stderr, + Err(e) => return read_err(ctx, a, e), + }; + let now_tip = match gw::read_ref(git, &d, &full) { + Ok(t) => t, + Err(e) => return read_err(ctx, a, e), + }; + let expected_old = if creating { None } else { old.as_deref() }; + // A killed update-ref may have landed and been built on by a push before the re-read: our + // commit is then an ancestor of the tip, and the write succeeded. + let landed = match &now_tip { + Some(t) if *t != commit => match gw::is_ancestor(git, &d, &commit, t) { + Ok(b) => b, + Err(e) => return read_err(ctx, a, e), + }, + _ => false, + }; + let seen = if landed { + Some(commit.as_str()) + } else { + now_tip.as_deref() + }; + match gw::classify(expected_old, bootstrap, &commit, seen, &failure) { + Outcome::Done => return published(ctx, a, resp), + Outcome::Lost => locked = false, + Outcome::LostAfterBackoff => { + locked = true; + std::thread::sleep(BACKOFF); + continue; + } + Outcome::Gone => return no_branch(ctx, &base_name), + Outcome::Exists => { + return unprocessable(ctx, &format!("branch already exists [name: {target_name}]")) + } + Outcome::DfConflict => { + return unprocessable( + ctx, + &format!("branch name conflicts with an existing branch [name: {target_name}]"), + ) + } + Outcome::Fatal => { + eprintln!("sc-forge: update-ref {full} failed on attempt {attempt}: {failure}"); + return ctx.app.internal(&"could not update the branch"); + } + } + eprintln!("sc-forge: contents write to {full} lost round {attempt}: {failure}"); + } + if locked { + // Three rounds against a ref lock that never cleared: nothing changed the branch. A killed + // git can leave `refs/heads/.lock` behind; it blocks pushes as well. + eprintln!( + "sc-forge: refs/heads/{target_name} of repository {} stayed locked; if no git process is running, remove its .lock file", + a.repo.id + ); + return error(&ctx.app.base, 503, "the branch is locked, try again later"); + } + error( + &ctx.app.base, + 409, + "the branch was changed concurrently, try again", + ) +} + +fn gobj_branches(ctx: &Ctx, a: &Access) -> Result, Resp> { + gw::branch_names(&ctx.app.life.git, &dir(ctx, a)).map_err(|e| read_err(ctx, a, e)) +} + +fn write_err(ctx: &Ctx, a: &Access, e: WriteErr) -> Resp { + match e { + WriteErr::Git(e) => read_err(ctx, a, e), + WriteErr::Rewrite(m) => { + eprintln!("sc-forge: contents write could not rewrite a directory: {m}"); + unprocessable(ctx, "a directory on the path cannot be rewritten") + } + } +} + +/// The ref moved: stamp the repository (log-only, the write is already public) and answer. +fn published(ctx: &Ctx, a: &Access, resp: Resp) -> Resp { + if let Err(e) = repos::touch(ctx.conn, a.repo.id, now()) { + eprintln!( + "sc-forge: could not stamp repository {} after a contents write: {e}", + a.repo.id + ); + } + resp +} + +/// The Gitea response for the commit just built. +#[allow(clippy::too_many_arguments)] +fn response( + ctx: &Ctx, + a: &Access, + w: &Write, + target: &str, + commit: &str, + tree: &str, + parent: Option<&str>, + blob: Option<&str>, + mode: &str, + when: i64, +) -> Resp { + let (api, web) = (api_base(ctx, a), web_base(ctx, a)); + let user = &ctx.who.user; + let email = commit_email(user); + let person = json!({"name": user.login, "email": email, "date": rfc3339(when)}); + let content = match blob { + Some(oid) => { + let e = TreeEntry { + mode: mode.to_string(), + kind: "blob".into(), + oid: oid.to_string(), + size: Some(w.bytes.len() as u64), + path: w.path.clone(), + }; + content_value( + ctx, + a, + target, + RefKind::Branch, + &e, + commit, + json!("base64"), + Value::String(b64::encode(&w.bytes)), + Value::Null, + ) + } + None => Value::Null, + }; + let body = json!({ + "content": content, + "commit": { + "url": format!("{api}/git/commits/{commit}"), + "sha": commit, + "created": rfc3339(when), + "html_url": format!("{web}/commit/{commit}"), + "author": person, + "committer": person, + "parents": parent.map(|p| vec![json!({"url": format!("{api}/git/commits/{p}"), "sha": p, "created": rfc3339(when)})]).unwrap_or_default(), + "message": w.message, + "tree": {"url": format!("{api}/git/trees/{tree}"), "sha": tree, "created": rfc3339(when)}, + }, + "verification": verification(), + }); + Resp::json(if w.verb == Verb::Post { 201 } else { 200 }, &body) +} diff --git a/crates/forge/src/gitproc.rs b/crates/forge/src/gitproc.rs index 84861a66..91bcb9aa 100644 --- a/crates/forge/src/gitproc.rs +++ b/crates/forge/src/gitproc.rs @@ -65,6 +65,9 @@ impl Out { pub struct Identity { name: String, email: String, + /// Commit time (unix seconds), when the caller needs the commit to match a response it has + /// already built; otherwise git uses the clock. + date: Option, } impl Identity { @@ -75,8 +78,15 @@ impl Identity { (safe(name) && safe(email)).then(|| Identity { name: name.into(), email: email.into(), + date: None, }) } + + /// The same identity with both commit dates fixed to `secs` (UTC). + pub fn at(mut self, secs: i64) -> Identity { + self.date = Some(secs); + self + } } /// One git invocation. @@ -183,6 +193,9 @@ impl Git { for who in ["AUTHOR", "COMMITTER"] { env.push((format!("GIT_{who}_NAME"), id.name.clone())); env.push((format!("GIT_{who}_EMAIL"), id.email.clone())); + if let Some(t) = id.date { + env.push((format!("GIT_{who}_DATE"), format!("{t} +0000"))); + } } } env diff --git a/crates/forge/src/gitwrite.rs b/crates/forge/src/gitwrite.rs new file mode 100644 index 00000000..4e221a81 --- /dev/null +++ b/crates/forge/src/gitwrite.rs @@ -0,0 +1,931 @@ +//! The git half of contents writes (F3b): path rules, and building a commit from an edited tree with +//! plumbing only. Nothing here moves a ref except [`update_ref`], which the caller runs last. Trees +//! are handled as raw bytes from `ls-tree -z` to `mktree -z`: a name is never decoded, so an entry +//! the request did not touch cannot be renamed by the rewrite. + +use crate::gitproc::{Git, GitError, Identity, Spec}; +use std::path::Path; +use std::time::{Duration, Instant}; + +/// Most path components a write may have (each level costs a few git processes). +pub const MAX_DEPTH: usize = 64; +/// Largest raw listing of one directory read for a write. +pub const MAX_LISTING: usize = 16 << 20; +/// Cumulative raw listings of one write attempt. +pub const TOTAL_LISTING: usize = 32 << 20; +/// The all-zero id: "the ref must not exist". +pub const ZERO: &str = "0000000000000000000000000000000000000000"; +/// Longest line `fsck` accepts in a `.gitattributes`. +const ATTR_LINE: usize = 2048; + +/// Code points HFS+ ignores when it compares names, so `.g\u{200c}it` is `.git` on a Mac. +fn hfs_ignorable(c: char) -> bool { + matches!(c, '\u{200c}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{206a}'..='\u{206f}' | '\u{feff}') +} + +/// The names `fsck` treats as the special file `.`: the dotted name, and on NTFS the 8.3 short +/// names `~1` to `~4` and `~1` to `~9`. `hash` is NTFS's fall-back short-name +/// prefix for that file (`gi7eba` for `.gitmodules`). +fn special_names(name: &str, hash: Option<&str>) -> Vec { + let mut v = vec![format!(".{name}")]; + v.extend((1..=4).map(|n| format!("{}~{n}", &name[..6]))); + if let Some(h) = hash { + v.extend((1..=9).map(|n| format!("{h}~{n}"))); + } + v +} + +/// Whether a path component is one of `names` on any filesystem git runs on. +fn is_alias(component: &str, names: &[String]) -> bool { + let c: String = component.chars().filter(|c| !hfs_ignorable(*c)).collect(); + // NTFS ignores trailing dots and spaces and treats `name:stream` as the file `name`. + let c = c.split(':').next().unwrap_or(""); + let c = c.trim_end_matches([' ', '.']).to_ascii_lowercase(); + names.contains(&c) +} + +fn git_names() -> Vec { + vec![".git".into(), "git~1".into()] +} + +fn modules_names() -> Vec { + special_names("gitmodules", Some("gi7eba")) +} + +fn attributes_names() -> Vec { + special_names("gitattributes", Some("gi7d29")) +} + +/// Whether a client may create or change `path` through the API. Stricter than the read rules: a path a +/// checkout on macOS or Windows would resolve into `.git`, and `.gitmodules` (whose content `fsck` +/// checks only on pushes), can never be written here. +pub fn write_path_ok(path: &str) -> bool { + let (git, modules, attrs) = (git_names(), modules_names(), attributes_names()); + let comps: Vec<&str> = path.split('/').collect(); + crate::gitobj::valid_path(path) + && !path.ends_with('/') + && !path.contains('\\') + && comps.len() <= MAX_DEPTH + && comps.iter().enumerate().all(|(i, c)| { + c.len() <= 255 + && !is_alias(c, &git) + && !is_alias(c, &modules) + // fsck requires `.gitattributes` to be a blob, so it cannot be a directory + && (i + 1 == comps.len() || !is_alias(c, &attrs)) + }) +} + +/// Whether `content` may be stored at `path`: `fsck` rejects a `.gitattributes` (by any of its aliases) +/// with a line of 2048 bytes or more, so a server-made commit must not carry one either. +pub fn content_ok(path: &str, content: &[u8]) -> bool { + let name = path.rsplit('/').next().unwrap_or(path); + !is_alias(name, &attributes_names()) + || content.split(|b| *b == b'\n').all(|l| l.len() < ATTR_LINE) +} + +/// One raw tree entry. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Entry { + /// As `ls-tree` prints it: `100644`, `100755`, `120000`, `160000` or `040000`. + pub mode: String, + /// `blob`, `tree` or `commit`. + pub kind: String, + pub oid: String, + pub name: Vec, +} + +/// Parse `ls-tree -z` output (no `-l`); `None` when a record is malformed. +pub fn parse_tree(raw: &[u8]) -> Option> { + let mut out = Vec::new(); + for rec in raw.split(|b| *b == 0).filter(|r| !r.is_empty()) { + let tab = rec.iter().position(|b| *b == b'\t')?; + let meta = std::str::from_utf8(&rec[..tab]).ok()?; + let mut f = meta.split(' '); + let (mode, kind, oid) = (f.next()?, f.next()?, f.next()?); + if f.next().is_some() { + return None; + } + out.push(Entry { + mode: mode.to_string(), + kind: kind.to_string(), + oid: oid.to_string(), + name: rec[tab + 1..].to_vec(), + }); + } + Some(out) +} + +/// The `mktree -z` input for `entries`. +pub fn mktree_input(entries: &[Entry]) -> Vec { + let mut b = Vec::new(); + for e in entries { + b.extend_from_slice(format!("{} {} {}\t", e.mode, e.kind, e.oid).as_bytes()); + b.extend_from_slice(&e.name); + b.push(0); + } + b +} + +/// Why a write could not be built or published (the handler maps these to responses). +#[derive(Debug)] +pub enum WriteErr { + Git(GitError), + /// `mktree` or the parser refused a directory that already exists in the repository. + Rewrite(String), +} + +impl From for WriteErr { + fn from(e: GitError) -> WriteErr { + WriteErr::Git(e) + } +} + +/// What one write attempt may still use: listing bytes and time. Every git command gets what is left +/// of the time (at most 60 s), so a deep path on slow storage cannot outlive the operation's deadline. +pub struct Budget { + pub listing: usize, + pub end: Instant, +} + +impl Budget { + pub fn new(end: Instant) -> Budget { + Budget { + listing: TOTAL_LISTING, + end, + } + } +} + +fn run( + git: &Git, + dir: &Path, + args: &[&str], + stdin: Option<&[u8]>, + id: Option<&Identity>, + cap: usize, + end: Instant, +) -> Result, GitError> { + let left = end.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(GitError::Timeout { stdout: Vec::new() }); + } + let out = git.run(&Spec { + repo: Some(dir), + args, + stdin, + identity: id, + max_output: Some(cap), + timeout: Some(left.min(Duration::from_secs(60))), + })?; + if out.ok() { + Ok(out.stdout) + } else { + Err(GitError::Failed( + String::from_utf8_lossy(&out.stderr).trim().to_string(), + )) + } +} + +fn read_tree( + git: &Git, + dir: &Path, + oid: &str, + budget: &mut Budget, +) -> Result, WriteErr> { + let cap = MAX_LISTING.min(budget.listing); + let raw = run( + git, + dir, + &["ls-tree", "-z", oid], + None, + None, + cap, + budget.end, + )?; + budget.listing -= raw.len().min(budget.listing); + parse_tree(&raw).ok_or_else(|| WriteErr::Rewrite(format!("unreadable tree {oid}"))) +} + +/// What the directories along a path hold. +pub struct Walk { + /// `levels[i]` lists the directory that should contain component `i`; it is as long as the path + /// when every parent directory exists. + pub levels: Vec>, + /// The first component that exists but is not a directory. + pub blocker: Option, +} + +impl Walk { + /// The existing entry at the end of `comps`, if the parents exist. + pub fn leaf(&self, comps: &[&str]) -> Option<&Entry> { + if self.levels.len() < comps.len() || self.blocker.is_some() { + return None; + } + let last = comps.last()?.as_bytes(); + self.levels[comps.len() - 1].iter().find(|e| e.name == last) + } +} + +/// Read the directories from `root` (`None`: an unborn branch, an empty root) down `comps`. +pub fn walk( + git: &Git, + dir: &Path, + root: Option<&str>, + comps: &[&str], + budget: &mut Budget, +) -> Result { + let mut levels = vec![match root { + Some(t) => read_tree(git, dir, t, budget)?, + None => Vec::new(), + }]; + let mut blocker = None; + for (i, c) in comps[..comps.len() - 1].iter().enumerate() { + match levels[i].iter().find(|e| e.name == c.as_bytes()) { + None => break, + Some(e) if e.kind == "tree" => { + let next = read_tree(git, dir, &e.oid, budget)?; + levels.push(next); + } + Some(_) => { + blocker = Some(i); + break; + } + } + } + Ok(Walk { levels, blocker }) +} + +/// What a write intends to do to the file. +pub enum Op { + Create, + /// Replace the blob with this id. + Update(String), + /// Remove the blob with this id. + Delete(String), +} + +/// Why the current tree does not allow the write. +#[derive(Debug, PartialEq, Eq)] +pub enum Refuse { + AlreadyExists, + /// A parent directory of the path is a file; carries the path up to it. + Blocked(String), + NoFile, + /// Not a regular file (symlink, submodule, directory). + NotRegular, + ShaMismatch { + given: String, + expected: String, + }, +} + +/// Check `op` against what `walk` found; on success the mode of the file being replaced. +pub fn check(op: &Op, comps: &[&str], walk: &Walk) -> Result, Refuse> { + if let Some(i) = walk.blocker { + return match op { + Op::Create => Err(Refuse::Blocked(comps[..=i].join("/"))), + _ => Err(Refuse::NoFile), + }; + } + let leaf = walk.leaf(comps); + match (op, leaf) { + (Op::Create, Some(_)) => Err(Refuse::AlreadyExists), + (Op::Create, None) => Ok(None), + (Op::Update(_) | Op::Delete(_), None) => Err(Refuse::NoFile), + (Op::Update(given) | Op::Delete(given), Some(e)) => { + if e.kind != "blob" || !matches!(e.mode.as_str(), "100644" | "100755") { + return Err(Refuse::NotRegular); + } + if &e.oid != given { + return Err(Refuse::ShaMismatch { + given: given.clone(), + expected: e.oid.clone(), + }); + } + Ok(Some(e.mode.clone())) + } + } +} + +fn mktree(git: &Git, dir: &Path, entries: &[Entry], end: Instant) -> Result { + let input = mktree_input(entries); + match run(git, dir, &["mktree", "-z"], Some(&input), None, 4096, end) { + Ok(o) => Ok(String::from_utf8_lossy(&o).trim().to_string()), + Err(GitError::Failed(m)) => Err(WriteErr::Rewrite(m)), + Err(e) => Err(e.into()), + } +} + +/// Rebuild the trees along `comps` with the file replaced by `file` (`mode`, blob id) or removed +/// (`None`); returns the new root tree. An emptied directory is dropped from its parent. +pub fn rebuild( + git: &Git, + dir: &Path, + walk: &Walk, + comps: &[&str], + file: Option<(&str, &str)>, + end: Instant, +) -> Result { + let n = comps.len(); + let mut child: Option = file.map(|(mode, oid)| Entry { + mode: mode.to_string(), + kind: "blob".into(), + oid: oid.to_string(), + name: comps[n - 1].as_bytes().to_vec(), + }); + let mut root = String::new(); + for i in (0..n).rev() { + let mut entries = walk.levels.get(i).cloned().unwrap_or_default(); + entries.retain(|e| e.name != comps[i].as_bytes()); + entries.extend(child.take()); + if entries.is_empty() && i > 0 { + continue; + } + let oid = mktree(git, dir, &entries, end)?; + if i == 0 { + root = oid; + } else { + child = Some(Entry { + mode: "040000".into(), + kind: "tree".into(), + oid, + name: comps[i - 1].as_bytes().to_vec(), + }); + } + } + Ok(root) +} + +/// Store `bytes` as a blob. +pub fn hash_blob(git: &Git, dir: &Path, bytes: &[u8], end: Instant) -> Result { + let o = run( + git, + dir, + &["hash-object", "-w", "--no-filters", "-t", "blob", "--stdin"], + Some(bytes), + None, + 4096, + end, + )?; + Ok(String::from_utf8_lossy(&o).trim().to_string()) +} + +/// A commit of `tree` on top of `parent`, as `id` (dates included), with `message` verbatim. +pub fn commit_tree( + git: &Git, + dir: &Path, + tree: &str, + parent: Option<&str>, + message: &str, + id: &Identity, + end: Instant, +) -> Result { + let mut args = vec!["commit-tree", tree]; + if let Some(p) = parent { + args.extend(["-p", p]); + } + let o = run( + git, + dir, + &args, + Some(message.as_bytes()), + Some(id), + 4096, + end, + )?; + Ok(String::from_utf8_lossy(&o).trim().to_string()) +} + +/// Is commit `a` reachable from commit `b`? (A write that landed and was then built on by someone else.) +pub fn is_ancestor(git: &Git, dir: &Path, a: &str, b: &str) -> Result { + let out = git.run(&Spec { + repo: Some(dir), + args: &["merge-base", "--is-ancestor", a, b], + ..Default::default() + })?; + match out.status.code() { + Some(0) => Ok(true), + Some(1) => Ok(false), + _ => Err(GitError::Failed( + String::from_utf8_lossy(&out.stderr).trim().to_string(), + )), + } +} + +/// Every branch name (without `refs/heads/`), for the collision check of a new branch. +pub fn branch_names(git: &Git, dir: &Path) -> Result, GitError> { + let args = ["for-each-ref", "--format=%(refname:lstrip=2)", "refs/heads"]; + let raw = run( + git, + dir, + &args, + None, + None, + MAX_LISTING, + Instant::now() + Duration::from_secs(60), + )?; + Ok(String::from_utf8_lossy(&raw) + .lines() + .map(String::from) + .collect()) +} + +/// The commit `full` (a `refs/heads/...` name) points at, `None` when it does not exist. +pub fn read_ref(git: &Git, dir: &Path, full: &str) -> Result, GitError> { + let out = git.run(&Spec { + repo: Some(dir), + args: &["rev-parse", "--verify", "-q", "--end-of-options", full], + ..Default::default() + })?; + match out.status.code() { + Some(0) => Ok(Some(out.text())), + Some(1) => Ok(None), + _ => Err(GitError::Failed( + String::from_utf8_lossy(&out.stderr).trim().to_string(), + )), + } +} + +/// Move `full` to `new` if it still is `expected` (`ZERO`: if it does not exist). `Ok(Some(stderr))` is a +/// refusal by git; the caller decides what it meant with [`classify`]. +pub fn update_ref( + git: &Git, + dir: &Path, + full: &str, + new: &str, + expected: &str, +) -> Result, GitError> { + match git.run(&Spec { + repo: Some(dir), + args: &["update-ref", full, new, expected], + ..Default::default() + }) { + Ok(o) if o.ok() => Ok(None), + Ok(o) => Ok(Some(String::from_utf8_lossy(&o.stderr).trim().to_string())), + // The process was killed: whether the ref moved is for the re-read to say. + Err(GitError::Timeout { .. }) => Ok(Some("update-ref timed out".into())), + Err(e) => Err(e), + } +} + +/// What a failed `update-ref` meant, judged from a re-read of the ref. +#[derive(Debug, PartialEq, Eq)] +pub enum Outcome { + /// The ref already holds the new commit (a killed process whose update landed). + Done, + /// Somebody else moved or created the ref: rebuild on its tip. + Lost, + /// A ref lock was held: rebuild after a short pause. + LostAfterBackoff, + /// The branch disappeared under an advance. + Gone, + /// An explicit `new_branch` that exists now. + Exists, + /// The new name collides with another branch name in the ref tree. + DfConflict, + Fatal, +} + +/// `expected_old` is the tip an advance started from, `None` for a creation; `bootstrap` marks the +/// creation of the default branch of an unborn repository (a lost race there is an advance, not a +/// duplicate `new_branch`). +pub fn classify( + expected_old: Option<&str>, + bootstrap: bool, + new: &str, + now: Option<&str>, + stderr: &str, +) -> Outcome { + if now == Some(new) { + return Outcome::Done; + } + let locked = stderr.contains(".lock") + && (stderr.contains("File exists") || stderr.contains("Unable to create")); + match (expected_old, now) { + (Some(_), None) => Outcome::Gone, + (Some(old), Some(cur)) if cur != old => Outcome::Lost, + (Some(_), Some(_)) | (None, None) if locked => Outcome::LostAfterBackoff, + (None, Some(_)) if bootstrap => Outcome::Lost, + (None, Some(_)) => Outcome::Exists, + (None, None) if stderr.contains("cannot lock ref") && stderr.contains("exists") => { + Outcome::DfConflict + } + _ => Outcome::Fatal, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::gitproc::find_git; + + /// Removes its directory when dropped, and checks it is gone (also when a test fails). + struct Tmp(std::path::PathBuf); + + impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + if !std::thread::panicking() { + assert!(!self.0.exists(), "test dir must be gone after teardown"); + } + } + } + + fn repo(tag: &str) -> (Git, Tmp, std::path::PathBuf) { + let d = + std::env::temp_dir().join(format!("scl-forge-gitwrite-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + let g = Git::new(&d, find_git().expect("git"), 4).unwrap(); + let dir = d.join("1.git"); + let init = [ + "init", + "--bare", + "--quiet", + "--template=", + "--initial-branch=main", + dir.to_str().unwrap(), + ]; + assert!(g + .run(&Spec { + args: &init, + ..Default::default() + }) + .unwrap() + .ok()); + (g, Tmp(d), dir) + } + + fn far() -> Instant { + Instant::now() + Duration::from_secs(60) + } + + fn entry(mode: &str, kind: &str, oid: &str, name: &[u8]) -> Entry { + Entry { + mode: mode.into(), + kind: kind.into(), + oid: oid.into(), + name: name.to_vec(), + } + } + + #[test] + fn ordinary_paths_pass() { + for p in [ + "a", + "dir/file.txt", + "a/b/c.md", + ".gitignore", + ".github/workflows/ci.yml", + "gitx", + "x.git", + "a.git/b", + ".gitattributes", + ] { + assert!(write_path_ok(p), "{p}"); + } + } + + #[test] + fn git_aliases_and_separators_are_refused() { + for p in [ + "", + "/a", + "a/", + "a//b", + "a/./b", + "a/../b", + ".git", + ".GIT", + "a/.git/hooks/x", + ".git.", + ".git ", + ".git. .", + "git~1/x", + "GIT~1", + ".g\u{200c}it/x", + ".git\u{feff}", + ".git::$INDEX_ALLOCATION", + ".git:stream", + "a\\b", + "..\\x", + "a\u{0}b", + "a\nb", + ".gitmodules", + "x/.GITMODULES", + ".gitmodules.", + "gitmod~1", + ".gitmodules::$DATA", + ".gitm\u{200d}odules", + "gitmod~2", + "GITMOD~4", + "gi7eba~1", + "gi7eba~9", + "d/gi7eba~5", + ".gitattributes/x", + "a/.GITATTRIBUTES/b", + "gitatt~1/f", + "gi7d29~3/f", + ] { + assert!(!write_path_ok(p), "{p:?}"); + } + assert!(!write_path_ok(&"x".repeat(256))); + assert!(write_path_ok(&"x".repeat(255))); + assert!(write_path_ok(&vec!["d"; MAX_DEPTH].join("/"))); + assert!(!write_path_ok(&vec!["d"; MAX_DEPTH + 1].join("/"))); + } + + #[test] + fn gitattributes_lines_are_bounded() { + assert!(content_ok("a/.gitattributes", &[b'x'; ATTR_LINE - 1])); + assert!(!content_ok("a/.gitattributes", &[b'x'; ATTR_LINE])); + assert!(!content_ok("a/.gitattributes", &[b'x'; ATTR_LINE + 1])); + for alias in [ + ".GITATTRIBUTES", + ".gitattributes.", + ".gitattributes ", + "gitatt~1", + "GITATT~4", + "gi7d29~1", + "gi7d29~9", + ] { + assert!(!content_ok(&format!("d/{alias}"), &[b'x'; 3000]), "{alias}"); + assert!(content_ok(&format!("d/{alias}"), b"ok\n"), "{alias}"); + } + assert!(content_ok("d/gitatt~5", &[b'x'; 3000]), "not an alias"); + assert!(content_ok("a/readme", &[b'x'; 10_000])); + } + + #[test] + fn raw_trees_round_trip_with_odd_names() { + let oid = "1".repeat(40); + let es = vec![ + entry("100644", "blob", &oid, b"plain"), + entry("100644", "blob", &oid, b"bad\xff\x01name"), + entry("040000", "tree", &oid, b"tab\tnl\nname"), + ]; + assert_eq!(parse_tree(&mktree_input(&es)).unwrap(), es); + assert!(parse_tree(b"100644 blob\n").is_none()); + } + + #[test] + fn what_the_tree_allows() { + let oid = "a".repeat(40); + let w = Walk { + levels: vec![ + vec![ + entry("040000", "tree", &oid, b"d"), + entry("100644", "blob", &oid, b"f"), + entry("120000", "blob", &oid, b"l"), + ], + vec![entry("100755", "blob", &oid, b"x")], + ], + blocker: None, + }; + assert_eq!(check(&Op::Create, &["d", "new"], &w), Ok(None)); + assert_eq!( + check(&Op::Create, &["d", "x"], &w), + Err(Refuse::AlreadyExists) + ); + assert_eq!( + check(&Op::Update(oid.clone()), &["d", "x"], &w), + Ok(Some("100755".into())) + ); + assert_eq!( + check(&Op::Update("b".repeat(40)), &["d", "x"], &w), + Err(Refuse::ShaMismatch { + given: "b".repeat(40), + expected: oid.clone() + }) + ); + assert_eq!( + check(&Op::Delete(oid.clone()), &["d", "none"], &w), + Err(Refuse::NoFile) + ); + let top = Walk { + levels: w.levels[..1].to_vec(), + blocker: None, + }; + assert_eq!( + check(&Op::Update(oid.clone()), &["l"], &top), + Err(Refuse::NotRegular) + ); + assert_eq!( + check(&Op::Update(oid.clone()), &["d"], &top), + Err(Refuse::NotRegular) + ); + let blocked = Walk { + levels: w.levels[..1].to_vec(), + blocker: Some(0), + }; + assert_eq!( + check(&Op::Create, &["f", "g"], &blocked), + Err(Refuse::Blocked("f".into())) + ); + assert_eq!( + check(&Op::Delete(oid), &["f", "g"], &blocked), + Err(Refuse::NoFile) + ); + } + + #[test] + fn a_failed_update_ref_is_judged_by_the_ref_afterwards() { + let (old, new, other) = ("o".repeat(40), "n".repeat(40), "x".repeat(40)); + let c = |expected: Option<&str>, boot: bool, now: Option<&str>, err: &str| { + classify(expected, boot, &new, now, err) + }; + assert_eq!(c(Some(&old), false, Some(&new), "killed"), Outcome::Done); + assert_eq!(c(Some(&old), false, None, "x"), Outcome::Gone); + assert_eq!( + c(Some(&old), false, Some(&other), "is at x but expected o"), + Outcome::Lost + ); + assert_eq!( + c( + Some(&old), + false, + Some(&old), + "Unable to create 'refs/heads/m.lock': File exists." + ), + Outcome::LostAfterBackoff + ); + assert_eq!( + c(Some(&old), false, Some(&old), "disk on fire"), + Outcome::Fatal + ); + assert_eq!( + c(None, false, Some(&other), "reference already exists"), + Outcome::Exists + ); + assert_eq!( + c(None, true, Some(&other), "reference already exists"), + Outcome::Lost + ); + assert_eq!( + c( + None, + false, + None, + "cannot lock ref 'refs/heads/a/b': 'refs/heads/a' exists; cannot create" + ), + Outcome::DfConflict + ); + assert_eq!( + c( + None, + false, + None, + "Unable to create 'refs/heads/a.lock': File exists." + ), + Outcome::LostAfterBackoff + ); + } + + #[test] + fn a_rewrite_keeps_every_untouched_entry_byte_for_byte() { + let (g, d, dir) = repo("rewrite"); + let blob = hash_blob(&g, &dir, b"old\n", far()).unwrap(); + let other = hash_blob(&g, &dir, b"other\n", far()).unwrap(); + let odd: [&[u8]; 3] = [b"bad\xff\x01name", b"tab\tnl\nname", b"z"]; + let mut dir_entries = vec![entry("100644", "blob", &blob, b"edit.txt")]; + dir_entries.extend(odd.iter().map(|n| entry("100755", "blob", &other, n))); + let sub = mktree(&g, &dir, &dir_entries, far()).unwrap(); + let root_entries = vec![ + entry("040000", "tree", &sub, b"d"), + entry("100644", "blob", &other, b"top"), + ]; + let root = mktree(&g, &dir, &root_entries, far()).unwrap(); + + let mut budget = Budget::new(far()); + let comps = ["d", "edit.txt"]; + let w = walk(&g, &dir, Some(&root), &comps, &mut budget).unwrap(); + assert!(check(&Op::Update(blob.clone()), &comps, &w).is_ok()); + let new_blob = hash_blob(&g, &dir, b"new\n", far()).unwrap(); + let new_root = rebuild(&g, &dir, &w, &comps, Some(("100644", &new_blob)), far()).unwrap(); + let mut budget = Budget::new(far()); + let w2 = walk(&g, &dir, Some(&new_root), &comps, &mut budget).unwrap(); + let listing = &w2.levels[1]; + for n in odd { + let e = listing + .iter() + .find(|e| e.name == n) + .unwrap_or_else(|| panic!("{n:?} survived")); + assert_eq!( + (e.mode.as_str(), e.oid.as_str()), + ("100755", other.as_str()) + ); + } + assert_eq!(w2.leaf(&comps).unwrap().oid, new_blob); + assert!(w2.levels[0].iter().any(|e| e.name == b"top")); + + // deleting the only file of a directory drops the directory + let root2 = mktree( + &g, + &dir, + &[ + entry( + "040000", + "tree", + &mktree(&g, &dir, &[entry("100644", "blob", &blob, b"only")], far()).unwrap(), + b"solo", + ), + entry("100644", "blob", &other, b"keep"), + ], + far(), + ) + .unwrap(); + let comps = ["solo", "only"]; + let mut budget = Budget::new(far()); + let w = walk(&g, &dir, Some(&root2), &comps, &mut budget).unwrap(); + let after = rebuild(&g, &dir, &w, &comps, None, far()).unwrap(); + let top = parse_tree( + &run( + &g, + &dir, + &["ls-tree", "-z", &after], + None, + None, + 4096, + far(), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + top.iter().map(|e| e.name.clone()).collect::>(), + vec![b"keep".to_vec()] + ); + // an unborn branch is an empty root; a new nested file builds its directories + let comps = ["a", "b", "c.txt"]; + let mut budget = Budget::new(far()); + let w = walk(&g, &dir, None, &comps, &mut budget).unwrap(); + assert_eq!(check(&Op::Create, &comps, &w), Ok(None)); + let t = rebuild(&g, &dir, &w, &comps, Some(("100644", &blob)), far()).unwrap(); + let listed = run( + &g, + &dir, + &["ls-tree", "-r", "-z", "--name-only", &t], + None, + None, + 4096, + far(), + ) + .unwrap(); + assert_eq!(listed, b"a/b/c.txt\0"); + drop(d); + } + + #[test] + fn refs_move_only_from_the_expected_tip() { + let (g, d, dir) = repo("refs"); + let blob = hash_blob(&g, &dir, b"x", far()).unwrap(); + let tree = mktree(&g, &dir, &[entry("100644", "blob", &blob, b"f")], far()).unwrap(); + let id = Identity::new("u", "u@example.invalid") + .unwrap() + .at(1_700_000_000); + let c1 = commit_tree(&g, &dir, &tree, None, "one\n", &id, far()).unwrap(); + let c2 = commit_tree(&g, &dir, &tree, Some(&c1), "two\n", &id, far()).unwrap(); + assert_eq!(read_ref(&g, &dir, "refs/heads/m").unwrap(), None); + assert_eq!( + update_ref(&g, &dir, "refs/heads/m", &c1, ZERO).unwrap(), + None + ); + assert!( + update_ref(&g, &dir, "refs/heads/m", &c1, ZERO) + .unwrap() + .is_some(), + "create twice" + ); + assert!( + update_ref(&g, &dir, "refs/heads/m", &c2, &c2) + .unwrap() + .is_some(), + "wrong expected tip" + ); + assert_eq!( + update_ref(&g, &dir, "refs/heads/m", &c2, &c1).unwrap(), + None + ); + assert_eq!( + read_ref(&g, &dir, "refs/heads/m").unwrap(), + Some(c2.clone()) + ); + assert!(is_ancestor(&g, &dir, &c1, &c2).unwrap()); + assert!(!is_ancestor(&g, &dir, &c2, &c1).unwrap()); + let msg = run( + &g, + &dir, + &["log", "-1", "--format=%an|%ae|%at|%cn|%ct|%s", &c2], + None, + None, + 4096, + far(), + ) + .unwrap(); + assert_eq!( + String::from_utf8_lossy(&msg).trim(), + "u|u@example.invalid|1700000000|u|1700000000|two" + ); + drop(d); + } +} diff --git a/crates/forge/src/lib.rs b/crates/forge/src/lib.rs index e08016a2..643967c2 100644 --- a/crates/forge/src/lib.rs +++ b/crates/forge/src/lib.rs @@ -13,10 +13,12 @@ pub mod auth; pub mod b64; pub mod cgi; pub mod content_api; +pub mod content_write; pub mod db; pub mod gitobj; pub mod gitproc; pub mod gitread; +pub mod gitwrite; pub mod hook; pub mod http; pub mod lifecycle; diff --git a/crates/forge/src/lifecycle.rs b/crates/forge/src/lifecycle.rs index b6eca72d..54c8080a 100644 --- a/crates/forge/src/lifecycle.rs +++ b/crates/forge/src/lifecycle.rs @@ -9,6 +9,13 @@ //! //! `serve` holds the [`DirLock`] for its lifetime, so no second process can reconcile or mutate //! the same directory, and a per-repository mutex serializes the server's own operations on one id. +//! +//! **Lock order** for a server-made ref write (contents writes now, merges and PR-ref updates in F4): +//! the repository's write gate first ([`Lifecycle::with_write_row`]: timed, 503 when busy), then the +//! shared lifecycle lock, each taken once. Nothing beneath that boundary may take the lifecycle lock +//! again (no `with_read` helper): the lock is writer-preferring, so a queued delete between two +//! shared acquisitions by one request deadlocks it. Never take the gate while holding the lifecycle +//! lock. use crate::gitproc::{Git, GitError, Identity, Spec}; use crate::refname; @@ -85,6 +92,15 @@ pub struct Report { pub missing_dirs: usize, } +/// How [`Lifecycle::with_write_row`] ended. +pub enum WriteRun { + /// The write gate stayed taken for the whole wait. + Busy, + /// The repository is deleted or being deleted. + Gone, + Ran(T), +} + /// Owns the on-disk layout and the per-repository serialization. pub struct Lifecycle { data_dir: PathBuf, @@ -92,6 +108,8 @@ pub struct Lifecycle { /// Live git transports by repository, cancelled by `delete` and at shutdown. pub registry: Arc, locks: Mutex>>>, + /// One writer per repository at a time; see the lock order above. + gates: Mutex>>, #[cfg(test)] crash_at: Mutex>, } @@ -113,6 +131,7 @@ impl Lifecycle { git, registry: Arc::default(), locks: Mutex::default(), + gates: Mutex::default(), #[cfg(test)] crash_at: Mutex::default(), }) @@ -136,6 +155,34 @@ impl Lifecycle { }) } + /// Run `f` (a server-made write to repository `id`) as its only writer, under the shared + /// lifecycle lock, with the repository's row as it is now (so the default branch is current). + /// `Busy`: another write held the gate for `wait`; `Gone`: the repository is not `live`. + pub fn with_write_row( + &self, + conn: &Connection, + id: i64, + wait: Duration, + f: impl FnOnce(&Repo) -> T, + ) -> Result, rusqlite::Error> { + let gate = self + .gates + .lock() + .unwrap_or_else(|e| e.into_inner()) + .entry(id) + .or_insert_with(|| Arc::new(crate::limits::Gate::new(1))) + .clone(); + let Some(_writer) = gate.acquire_timeout(wait) else { + return Ok(WriteRun::Busy); + }; + let lock = self.lock_for(id); + let _shared = lock.read().unwrap_or_else(|e| e.into_inner()); + Ok(match repos::find_by_id(conn, id)? { + Some(r) if r.state == "live" => WriteRun::Ran(f(&r)), + _ => WriteRun::Gone, + }) + } + /// `repos/.git`: always built from the integer id. pub fn repo_dir(&self, id: i64) -> PathBuf { self.data_dir.join("repos").join(format!("{id}.git")) @@ -362,6 +409,10 @@ impl Lifecycle { .lock() .unwrap_or_else(|e| e.into_inner()) .remove(&repo.id); + self.gates + .lock() + .unwrap_or_else(|e| e.into_inner()) + .remove(&repo.id); Ok(()) } diff --git a/crates/forge/src/policy.rs b/crates/forge/src/policy.rs index 890690bf..5066d0c4 100644 --- a/crates/forge/src/policy.rs +++ b/crates/forge/src/policy.rs @@ -38,6 +38,14 @@ pub fn check_update(name: &str, new: &str, default_branch: &str) -> Result<(), R )) } +/// May a contents write put a commit on branch `target`? `creating` is true when the write creates +/// the branch (`new_branch`, or the first commit of an unborn default). Called before the objects +/// are built and again immediately before the ref update, on every attempt, so a rule added in F4 +/// is evaluated at the moment the ref moves (the spec's linearization point). +pub fn check_contents_write(_target: &str, _creating: bool) -> Result<(), Refusal> { + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/forge/src/routes.rs b/crates/forge/src/routes.rs index c16fa228..c4c95de0 100644 --- a/crates/forge/src/routes.rs +++ b/crates/forge/src/routes.rs @@ -9,7 +9,7 @@ use crate::lifecycle::Lifecycle; use crate::limits::{Backoff, Gate, Proxies}; use crate::repos::{self, Access, Perm}; use crate::scope::{self, Cat, Level, Need}; -use crate::{api, content_api, db, repo_api, store}; +use crate::{api, content_api, content_write, db, repo_api, store}; use rusqlite::Connection; use std::path::PathBuf; @@ -419,15 +419,14 @@ pub static ROUTES: &[Route] = &[ Repo(Perm::Read), content_api::get_contents, ), - // F3b builds the writes; until then they answer 501 after the scope and permission checks, so - // a read-only token still gets its recorded 403. + // Contents writes (F3b): write scope and write permission, then a server-made commit. r( "POST", "/repos/{o}/{r}/contents/{p..}", Token, n(Cat::Repository, Level::Write), Repo(Perm::Write), - api::not_implemented, + content_write::write_contents, ), r( "PUT", @@ -435,7 +434,7 @@ pub static ROUTES: &[Route] = &[ Token, n(Cat::Repository, Level::Write), Repo(Perm::Write), - api::not_implemented, + content_write::write_contents, ), r( "DELETE", @@ -443,7 +442,7 @@ pub static ROUTES: &[Route] = &[ Token, n(Cat::Repository, Level::Write), Repo(Perm::Write), - api::not_implemented, + content_write::write_contents, ), r( "GET", diff --git a/crates/forge/tests/e2e_write.rs b/crates/forge/tests/e2e_write.rs new file mode 100644 index 00000000..b998561d --- /dev/null +++ b/crates/forge/tests/e2e_write.rs @@ -0,0 +1,782 @@ +//! F3b: contents writes (`POST`/`PUT`/`DELETE`), checked with real git against the bare repository. +#![cfg(unix)] + +mod common; +use common::*; +use serde_json::{json, Value}; + +const ZERO: &str = "0000000000000000000000000000000000000000"; + +struct W { + s: Srv, + auth: String, + id: i64, +} + +/// A repository `siteadmin/w` with the auto-init README commit on `main`. +fn world() -> W { + let s = Srv::start(); + let auth = token(&s.admin); + let r = s.post( + "/api/v1/user/repos", + &auth, + json!({"name": "w", "default_branch": "main", "auto_init": true}), + ); + assert_eq!(r.status, 201, "{}", r.msg()); + let id = r.json()["id"].as_i64().unwrap(); + W { s, auth, id } +} + +impl W { + fn url(&self, path: &str) -> String { + format!("/api/v1/repos/siteadmin/w/contents/{path}") + } + fn write(&self, method: &str, path: &str, body: Value) -> Resp { + self.s + .send(method, &self.url(path), Some(&self.auth), Some(&body)) + } + fn git(&self, args: &[&str]) -> String { + self.s.git(self.id, args, None) + } + fn tip(&self, branch: &str) -> String { + self.git(&["rev-parse", &format!("refs/heads/{branch}")]) + } + /// `fsck --strict` finds nothing wrong; unreferenced objects (a refused or lost write leaves some) are fine. + fn fsck(&self) { + assert_clean(&self.git(&["fsck", "--strict"])); + } + fn blob(&self, spec: &str) -> String { + self.git(&["rev-parse", spec]) + } +} + +fn assert_clean(fsck: &str) { + let bad: Vec<&str> = fsck + .lines() + .filter(|l| !l.starts_with("dangling ")) + .collect(); + assert!(bad.is_empty(), "fsck: {bad:?}"); +} + +fn enc(s: &str) -> String { + b64(s) +} + +#[test] +fn create_update_delete_round_trip_with_the_recorded_answers() { + let w = world(); + let before = w.tip("main"); + let r = w.write( + "POST", + "new.txt", + json!({"content": enc("new\n"), "message": "add new.txt", "branch": "main"}), + ); + assert_eq!(r.status, 201, "{}", r.msg()); + let j = r.json(); + let c1 = w.tip("main"); + assert_ne!(c1, before); + assert_eq!(j["commit"]["sha"], c1.as_str()); + assert_eq!(j["commit"]["message"], "add new.txt\n"); + assert_eq!(j["commit"]["parents"][0]["sha"], before.as_str()); + assert_eq!( + j["commit"]["tree"]["sha"], + w.blob("main^{tree}").as_str(), + "the real tree id" + ); + assert_eq!(j["content"]["name"], "new.txt"); + assert_eq!(j["content"]["sha"], w.blob("main:new.txt").as_str()); + assert_eq!(j["content"]["last_commit_sha"], c1.as_str()); + assert_eq!(j["content"]["content"], enc("new\n").as_str()); + assert_eq!( + j["content"]["url"] + .as_str() + .unwrap() + .rsplit_once("?ref=") + .unwrap() + .1, + "main" + ); + assert_eq!(j["verification"]["verified"], false); + assert_eq!(j["commit"]["author"]["name"], "siteadmin"); + assert_eq!( + w.git(&["log", "-1", "--format=%an|%cn|%ae"],), + format!( + "siteadmin|siteadmin|{}", + j["commit"]["author"]["email"].as_str().unwrap() + ) + ); + // the F3a read sees it + let got = w.s.get(&w.url("new.txt"), &w.auth); + assert_eq!(got.json()["sha"], j["content"]["sha"]); + + let dup = w.write( + "POST", + "new.txt", + json!({"content": enc("new\n"), "message": "again", "branch": "main"}), + ); + assert_eq!( + (dup.status, dup.msg().as_str()), + (422, "repository file already exists [path: new.txt]") + ); + + let sha = j["content"]["sha"].as_str().unwrap().to_string(); + let up = w.write( + "PUT", + "new.txt", + json!({"content": enc("two\n"), "message": "update", "sha": sha, "branch": "main"}), + ); + assert_eq!(up.status, 200, "{}", up.msg()); + let new_sha = up.json()["content"]["sha"].as_str().unwrap().to_string(); + assert_eq!(w.git(&["show", "main:new.txt"]), "two"); + + let stale = w.write( + "PUT", + "new.txt", + json!({"content": enc("x\n"), "message": "stale", "sha": sha, "branch": "main"}), + ); + assert_eq!(stale.status, 422); + assert_eq!( + stale.msg(), + format!("sha does not match [given: {sha}, expected: {new_sha}]") + ); + let nosha = w.write( + "PUT", + "new.txt", + json!({"content": enc("x\n"), "message": "no sha"}), + ); + assert_eq!( + (nosha.status, nosha.msg().as_str()), + (422, "[SHA]: Required") + ); + + let del = w.write( + "DELETE", + "new.txt", + json!({"message": "rm", "sha": new_sha}), + ); + assert_eq!(del.status, 200, "{}", del.msg()); + assert_eq!(del.json()["content"], Value::Null); + assert_eq!(w.s.get(&w.url("new.txt"), &w.auth).status, 404); + let again = w.write( + "DELETE", + "new.txt", + json!({"message": "rm", "sha": new_sha}), + ); + assert_eq!(again.status, 404); + w.fsck(); + assert!(!w.git(&["log", "--format=%s", "main"]).contains("again")); +} + +#[test] +fn nested_files_make_directories_and_deleting_the_last_one_removes_them() { + let w = world(); + let r = w.write( + "POST", + "a/b/c.txt", + json!({"content": enc("c\n"), "message": "nested"}), + ); + assert_eq!(r.status, 201, "{}", r.msg()); + assert_eq!( + w.git(&["ls-tree", "-r", "--name-only", "main"]), + "README.md\na/b/c.txt" + ); + let sha = r.json()["content"]["sha"].as_str().unwrap().to_string(); + let r = w.write("DELETE", "a/b/c.txt", json!({"sha": sha})); + assert_eq!(r.status, 200, "{}", r.msg()); + assert_eq!( + w.git(&["ls-tree", "-r", "--name-only", "main"]), + "README.md" + ); + assert_eq!( + w.git(&["log", "-1", "--format=%s", "main"]), + "Delete a/b/c.txt", + "default message" + ); + w.fsck(); +} + +#[test] +fn new_branch_leaves_the_base_alone_and_collisions_are_422() { + let w = world(); + let base = w.tip("main"); + let r = w.write( + "POST", + "f.txt", + json!({"content": enc("f\n"), "message": "to new", "branch": "main", "new_branch": "nb"}), + ); + assert_eq!(r.status, 201, "{}", r.msg()); + assert_eq!(w.tip("main"), base); + assert_eq!(w.git(&["rev-parse", "nb~1"]), base); + assert!(r.json()["content"]["url"] + .as_str() + .unwrap() + .ends_with("?ref=nb")); + let again = w.write( + "POST", + "g.txt", + json!({"content": enc("g\n"), "branch": "main", "new_branch": "nb"}), + ); + assert_eq!( + (again.status, again.msg().as_str()), + (422, "branch already exists [name: nb]") + ); + for n in ["nb/x", "feat", "x y", "-bad"] { + let r = w.write( + "POST", + "h.txt", + json!({"content": enc("h\n"), "branch": "main", "new_branch": n}), + ); + assert!( + matches!((n, r.status), ("feat", 201) | (_, 422)), + "{n}: {} {}", + r.status, + r.msg() + ); + } + let under = w.write( + "POST", + "i.txt", + json!({"content": enc("i\n"), "branch": "main", "new_branch": "feat/sub"}), + ); + assert_eq!( + under.status, + 422, + "feat exists as a branch: {}", + under.msg() + ); + // new_branch equal to branch, or empty, is the branch itself + let rel = w.write( + "POST", + "r.txt", + json!({"content": enc("r\n"), "branch": "main", "new_branch": "rel/1"}), + ); + assert_eq!(rel.status, 201, "{}", rel.msg()); + let above = w.write( + "POST", + "s.txt", + json!({"content": enc("s\n"), "branch": "main", "new_branch": "rel"}), + ); + assert_eq!( + above.status, + 422, + "rel is a directory of rel/1: {}", + above.msg() + ); + let dflt = w.write( + "POST", + "t.txt", + json!({"content": enc("t\n"), "new_branch": "main"}), + ); + assert_eq!( + dflt.status, + 201, + "new_branch equal to the default is the default: {}", + dflt.msg() + ); + let same = w.write( + "POST", + "j.txt", + json!({"content": enc("j\n"), "branch": "main", "new_branch": "main"}), + ); + assert_eq!(same.status, 201, "{}", same.msg()); + let empty = w.write( + "POST", + "k.txt", + json!({"content": enc("k\n"), "new_branch": ""}), + ); + assert_eq!(empty.status, 201); + let missing = w.write( + "POST", + "m.txt", + json!({"content": enc("m\n"), "branch": "nope"}), + ); + assert_eq!( + (missing.status, missing.json()["errors"][0].as_str()), + (404, Some("branch does not exist [name: nope]")) + ); + w.fsck(); +} + +#[test] +fn paths_that_a_checkout_could_misread_are_refused() { + let w = world(); + for p in [ + ".git/config", + "a/.GIT/x", + ".gitmodules", + "x/.git.", + "git~1/y", + "a%5Cb", + "a%2F%2Fb", + ] { + let r = w.write("POST", p, json!({"content": enc("x\n"), "message": "m"})); + assert_eq!(r.status, 422, "{p}: {}", r.msg()); + } + let tip = w.tip("main"); + assert_eq!( + w.write("POST", "ok.txt", json!({"content": "!!!", "message": "m"})) + .status, + 422, + "base64" + ); + assert_eq!( + w.write("POST", "ok.txt", json!({"message": "m"})).status, + 422, + "content required" + ); + assert_eq!( + w.write( + "POST", + "ok.txt", + json!({"content": enc("x"), "message": "a\u{0}b"}) + ) + .status, + 422, + "NUL" + ); + assert_eq!( + w.write( + "PUT", + "README.md", + json!({"content": enc("x"), "sha": "1", "from_path": "old"}) + ) + .status, + 422 + ); + let long = format!("{}\n", "x".repeat(3000)); + assert_eq!( + w.write("POST", ".gitattributes", json!({"content": enc(&long)})) + .status, + 422 + ); + assert_eq!( + w.write("POST", "empty.txt", json!({"content": ""})).status, + 201, + "an empty file is fine" + ); + assert_ne!(w.tip("main"), tip); + w.fsck(); +} + +#[test] +fn directories_files_and_modes_are_respected() { + let w = world(); + // build main with an executable, a symlink and a directory, using git directly + let blob = w.s.git( + w.id, + &["hash-object", "-w", "--stdin"], + Some(b"#!/bin/sh\n"), + ); + let link = + w.s.git(w.id, &["hash-object", "-w", "--stdin"], Some(b"run.sh")); + let sub = w.s.git( + w.id, + &["mktree"], + Some(format!("100644 blob {blob}\tinner\n").as_bytes()), + ); + let gitlink = "9".repeat(40); + let tree = w.s.git( + w.id, + &["mktree"], + Some( + format!( + "100755 blob {blob}\trun.sh\n120000 blob {link}\tlink\n040000 tree {sub}\tdir\n160000 commit {gitlink}\tsm\n" + ) + .as_bytes(), + ), + ); + let c = w.s.git( + w.id, + &["commit-tree", &tree, "-p", &w.tip("main")], + Some(b"fixture\n"), + ); + w.git(&["update-ref", "refs/heads/main", &c]); + // PUT keeps the executable bit + let r = w.write( + "PUT", + "run.sh", + json!({"content": enc("#!/bin/sh\necho\n"), "sha": blob}), + ); + assert_eq!(r.status, 200, "{}", r.msg()); + assert!(w.git(&["ls-tree", "main", "run.sh"]).starts_with("100755 ")); + // a symlink, a directory: refused + let r = w.write("PUT", "link", json!({"content": enc("x"), "sha": link})); + assert_eq!(r.status, 422, "{}", r.msg()); + let r = w.write("PUT", "dir", json!({"content": enc("x"), "sha": sub})); + assert_eq!(r.status, 422, "{}", r.msg()); + assert_eq!(w.write("DELETE", "link", json!({"sha": link})).status, 422); + assert_eq!( + w.write("PUT", "sm", json!({"content": enc("x"), "sha": gitlink})) + .status, + 422, + "submodule" + ); + assert_eq!(w.write("DELETE", "sm", json!({"sha": gitlink})).status, 422); + assert_eq!( + w.write("POST", "sm/x", json!({"content": enc("x")})).status, + 422, + "under a submodule" + ); + assert_eq!( + w.git(&["ls-tree", "main", "sm"]), + format!("160000 commit {gitlink}\tsm"), + "gitlink kept" + ); + // exists as a directory / a file is a parent + assert_eq!( + w.write("POST", "dir", json!({"content": enc("x")})).status, + 422 + ); + let r = w.write("POST", "run.sh/child", json!({"content": enc("x")})); + assert_eq!(r.status, 422, "{}", r.msg()); + assert_eq!( + w.write("DELETE", "run.sh/child", json!({"sha": blob})) + .status, + 404 + ); + w.fsck(); +} + +#[test] +fn untouched_siblings_keep_their_exact_names() { + let w = world(); + let blob = + w.s.git(w.id, &["hash-object", "-w", "--stdin"], Some(b"x\n")); + let mut input = Vec::new(); + for name in [ + &b"bad\xff\x01name"[..], + &b"tab\tnl\nname"[..], + &b"edit.txt"[..], + ] { + input.extend_from_slice(format!("100644 blob {blob}\t").as_bytes()); + input.extend_from_slice(name); + input.push(0); + } + let sub = w.s.git(w.id, &["mktree", "-z"], Some(&input)); + let tree = w.s.git( + w.id, + &["mktree"], + Some(format!("040000 tree {sub}\td\n").as_bytes()), + ); + let c = w.s.git(w.id, &["commit-tree", &tree], Some(b"odd names\n")); + w.git(&["update-ref", "refs/heads/odd", &c]); + let before = w.git(&["ls-tree", "-z", "odd:d"]); + let r = w.write( + "PUT", + "d/edit.txt", + json!({"content": enc("new\n"), "sha": blob, "branch": "odd"}), + ); + assert_eq!(r.status, 200, "{}", r.msg()); + let after = w.git(&["ls-tree", "-z", "odd:d"]); + // every untouched name is still there with its id; only edit.txt differs + let strip = |s: &str| { + s.split('\0') + .filter(|e| !e.ends_with("edit.txt")) + .map(String::from) + .collect::>() + }; + assert_eq!( + strip(&before), + strip(&after), + "raw listings of the untouched entries are equal" + ); + w.fsck(); +} + +#[test] +fn the_first_commit_of_an_empty_repository_and_an_unborn_default() { + let s = Srv::start(); + let auth = token(&s.admin); + let r = s.post( + "/api/v1/user/repos", + &auth, + json!({"name": "e", "default_branch": "main"}), + ); + let id = r.json()["id"].as_i64().unwrap(); + let put = |p: &str, v: Value| { + s.send( + "PUT", + &format!("/api/v1/repos/siteadmin/e/contents/{p}"), + Some(&auth), + Some(&v), + ) + }; + let post = |p: &str, v: Value| { + s.send( + "POST", + &format!("/api/v1/repos/siteadmin/e/contents/{p}"), + Some(&auth), + Some(&v), + ) + }; + assert_eq!( + put("a", json!({"content": enc("x"), "sha": "1"})).status, + 404, + "PUT on an unborn branch" + ); + assert_eq!( + post("a", json!({"content": enc("x"), "new_branch": "nb"})).status, + 404, + "new_branch from an unborn base" + ); + let r = post( + "README.md", + json!({"content": enc("# e\n"), "message": "seed"}), + ); + assert_eq!(r.status, 201, "{}", r.msg()); + assert_eq!(r.json()["commit"]["parents"], json!([])); + assert_eq!(s.git(id, &["rev-list", "--count", "main"], None), "1"); + assert_clean(&s.git(id, &["fsck", "--strict"], None)); + // an unborn default with another branch present: the seed still lands on the default branch + let r = s.post( + "/api/v1/user/repos", + &auth, + json!({"name": "f", "default_branch": "main"}), + ); + let fid = r.json()["id"].as_i64().unwrap(); + let tree = s.git(fid, &["mktree"], Some(b"")); + let c = s.git(fid, &["commit-tree", &tree], Some(b"x\n")); + s.git(fid, &["update-ref", "refs/heads/feat", &c], None); + let r = s.send( + "POST", + "/api/v1/repos/siteadmin/f/contents/seed.txt", + Some(&auth), + Some(&json!({"content": enc("s")})), + ); + assert_eq!(r.status, 201, "{}", r.msg()); + assert_eq!(s.git(fid, &["rev-list", "--count", "main"], None), "1"); + assert_eq!(s.git(fid, &["rev-parse", "feat"], None), c); +} + +#[test] +fn access_and_scope_are_checked_before_anything_is_written() { + let s = Srv::start(); + let auth = token(&s.admin); + s.post( + "/api/v1/user/repos", + &auth, + json!({"name": "w", "auto_init": true, "default_branch": "main"}), + ); + let (pr, pw, po) = ( + s.mk_user("reader"), + s.mk_user("writer"), + s.mk_user("outsider"), + ); + for (u, m) in [("reader", "read"), ("writer", "write")] { + let r = s.send( + "PUT", + &format!("/api/v1/repos/siteadmin/w/collaborators/{u}"), + Some(&auth), + Some(&json!({"permission": m})), + ); + assert_eq!(r.status, 204, "{}", r.msg()); + } + let id = 1; + let tip = s.git(id, &["rev-parse", "main"], None); + let post = |tok: &str| { + s.send( + "POST", + "/api/v1/repos/siteadmin/w/contents/x.txt", + Some(&token(tok)), + Some(&json!({"content": enc("x")})), + ) + }; + let reader = s.mk_token("reader", &pr, "t", &["all"]); + assert_eq!(post(&reader).status, 403, "read collaborator"); + let outsider = s.mk_token("outsider", &po, "t", &["all"]); + assert_eq!( + post(&outsider).status, + 404, + "no access looks like no repository" + ); + let ro = s.mk_token("writer", &pw, "ro", &["read:repository"]); + let r = post(&ro); + assert_eq!(r.status, 403); + assert!(r.msg().contains("write:repository"), "{}", r.msg()); + assert_eq!( + s.git(id, &["rev-parse", "main"], None), + tip, + "nothing was written" + ); + let writer = s.mk_token("writer", &pw, "rw", &["all"]); + assert_eq!(post(&writer).status, 201); + let anon = s.send( + "POST", + "/api/v1/repos/siteadmin/w/contents/y.txt", + None, + Some(&json!({"content": enc("y")})), + ); + assert_eq!(anon.status, 403, "the recorded anonymous answer"); +} + +#[test] +fn a_busy_runner_is_503_and_the_branch_does_not_move() { + let s = Srv::start_git(1, std::time::Duration::from_millis(100)); + let auth = token(&s.admin); + s.post( + "/api/v1/user/repos", + &auth, + json!({"name": "w", "auto_init": true, "default_branch": "main"}), + ); + let tip = s.git(1, &["rev-parse", "main"], None); + let busy = s.runner.occupy_for_tests(); + let r = s.send( + "POST", + "/api/v1/repos/siteadmin/w/contents/x.txt", + Some(&auth), + Some(&json!({"content": enc("x")})), + ); + assert_eq!(r.status, 503, "{}", r.msg()); + drop(busy); + assert_eq!(s.git(1, &["rev-parse", "main"], None), tip); +} + +#[test] +fn concurrent_api_writers_and_an_independent_ref_writer_lose_nothing() { + let w = std::sync::Arc::new(world()); + let racer = { + let w = w.clone(); + std::thread::spawn(move || { + // an independent writer (a stand-in for a push): adds a file on top of the tip, by CAS + let mut landed = Vec::new(); + for i in 0..15 { + let tip = w.tip("main"); + let listing = w.git(&["ls-tree", &format!("{tip}^{{tree}}")]); + let blob = w.s.git( + w.id, + &["hash-object", "-w", "--stdin"], + Some(format!("ext {i}\n").as_bytes()), + ); + let tree = w.s.git( + w.id, + &["mktree"], + Some(format!("{listing}\n100644 blob {blob}\text{i}.txt\n").as_bytes()), + ); + let c = w.s.git( + w.id, + &["commit-tree", &tree, "-p", &tip], + Some(format!("racer {i}\n").as_bytes()), + ); + let out = std::process::Command::new("git") + .args([ + "--git-dir", + w.s.dir + .join("repos") + .join(format!("{}.git", w.id)) + .to_str() + .unwrap(), + "update-ref", + "refs/heads/main", + &c, + &tip, + ]) + .output() + .unwrap(); + if out.status.success() { + landed.push((c, format!("ext{i}.txt"))); + } + std::thread::sleep(std::time::Duration::from_millis(3)); + } + landed + }) + }; + let writers: Vec<_> = (0..12) + .map(|i| { + let w = w.clone(); + std::thread::spawn(move || { + let r = w.write( + "POST", + &format!("f{i}.txt"), + json!({"content": enc(&format!("{i}\n")), "message": format!("w{i}")}), + ); + (i, r.status) + }) + }) + .collect(); + let results: Vec<(i32, u16)> = writers.into_iter().map(|h| h.join().unwrap()).collect(); + let landed = racer.join().unwrap(); + for (i, st) in &results { + assert!(matches!(st, 201 | 409), "writer {i}: {st}"); + } + let files = w.git(&["ls-tree", "--name-only", "main"]); + for (i, st) in &results { + assert_eq!( + files.contains(&format!("f{i}.txt")), + *st == 201, + "f{i}.txt vs {st}" + ); + } + assert!( + results.iter().filter(|(_, s)| *s == 201).count() >= 6, + "most writers land: {results:?}" + ); + assert!( + !landed.is_empty(), + "the independent writer landed something" + ); + for (c, file) in landed { + assert!( + files.contains(&file), + "{file} written outside the API survived the API's rebuilds" + ); + let out = std::process::Command::new("git") + .args([ + "--git-dir", + w.s.dir + .join("repos") + .join(format!("{}.git", w.id)) + .to_str() + .unwrap(), + "merge-base", + "--is-ancestor", + &c, + "main", + ]) + .output() + .unwrap(); + assert!(out.status.success(), "racer commit {c} was lost"); + } + w.fsck(); + let _ = ZERO; +} + +#[test] +fn deleting_the_repository_while_writes_are_queued_does_not_hang() { + let w = std::sync::Arc::new(world()); + let writers: Vec<_> = (0..8) + .map(|i| { + let w = w.clone(); + std::thread::spawn(move || { + w.write("POST", &format!("d{i}.txt"), json!({"content": enc("x\n")})) + .status + }) + }) + .collect(); + std::thread::sleep(std::time::Duration::from_millis(10)); + let del = + w.s.send("DELETE", "/api/v1/repos/siteadmin/w", Some(&w.auth), None); + assert_eq!(del.status, 204, "{}", del.msg()); + for h in writers { + let st = h.join().unwrap(); + assert!(matches!(st, 201 | 404 | 503), "{st}"); + } +} + +#[test] +fn a_directory_cannot_take_a_name_that_must_be_a_blob() { + let w = world(); + for p in [".gitattributes/x", "d/.GITATTRIBUTES/y", "gitatt~1/z"] { + let r = w.write("POST", p, json!({"content": enc("x\n")})); + assert_eq!(r.status, 422, "{p}: {}", r.msg()); + } + assert_eq!( + w.write( + "POST", + "d/.gitattributes", + json!({"content": enc("*.rs text\n")}) + ) + .status, + 201 + ); + w.fsck(); +} diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index 61bc9ffc..bc4b0e60 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -968,6 +968,30 @@ Threat-model pass #2 for the read endpoints. Tests are in `crates/forge/tests/e2 - **Not covered.** Content is served as stored: a file in a repository is returned base64-encoded in JSON, and raw/web URLs are not served, so no response is rendered as HTML. +### Content API writes (F3b) + +Threat-model pass #2, first half (server-side writes; merge races and the protection bypass tests come with F4). +Tests are in `crates/forge/tests/e2e_write.rs` and the `gitwrite` unit tests. + +- **Server-made commits bypass `receive.fsckObjects`.** The API therefore refuses what fsck would refuse on a push: + `.git` aliases (including NTFS and HFS spellings), `.gitmodules` (any content: the URL/path/name checks are not + reproduced), a `.gitattributes` (any alias, including `gitatt~1` and `gi7d29~1`) line of 2048 bytes or more, `.gitmodules` aliases (`gitmod~1` to `~4`, `gi7eba~1` to `~9`), backslashes, and paths deeper than 64 or with a component over + 255 bytes (`git_aliases_and_separators_are_refused`, `paths_that_a_checkout_could_misread_are_refused`). Names that only + a case-insensitive or Windows checkout would collide (device names, `a:b`, `Foo` beside `foo/`) are not prevented. +- **A write cannot damage what it did not touch.** Trees are rewritten from raw bytes; the entries the request did not name + keep their names and ids (`a_rewrite_keeps_every_untouched_entry_byte_for_byte`, `untouched_siblings_keep_their_exact_names`). + Legacy modes in a rebuilt directory are normalised. +- **Concurrent writes.** The ref moves only by compare-and-swap; a lost round rebuilds from the new tip and re-checks the + precondition (`concurrent_api_writers_and_an_independent_ref_writer_lose_nothing`: an independent real-git writer races + twelve API writers and loses no commit). A push is not excluded by the write gate, so a writer can get 409 under sustained + pushes. +- **Resource use.** One write at a time per repository (503 after 10 s); at most 3 attempts, 64 levels and 32 MiB of listings + per attempt, and 60 s for the whole operation; disk is reserved from the push pool (the content plus slack up front, then per attempt the bytes of the tree listings read). A write holds the shared lifecycle lock, so a delete waits for it + (`deleting_the_repository_while_writes_are_queued_does_not_hang`). +- **Identity cannot be chosen.** `author`, `committer` and `dates` in a request are ignored; the commit is the + authenticated user's. A write collaborator can still push any identity over git, as in Gitea. +- **Unreferenced objects.** A refused or lost write leaves blobs/trees/commits no ref reaches; automatic gc is off (F5 adds gc). + ## Reporting Security reports go to **toni@git-agentic.com** or GitHub private vulnerability diff --git a/docs/superpowers/plans/2026-10-24-p38-f3b-contents-writes.md b/docs/superpowers/plans/2026-10-24-p38-f3b-contents-writes.md new file mode 100644 index 00000000..7553f94f --- /dev/null +++ b/docs/superpowers/plans/2026-10-24-p38-f3b-contents-writes.md @@ -0,0 +1,131 @@ +# P38 F3b: contents writes (plan) + +`POST`, `PUT` and `DELETE` on `/repos/{o}/{r}/contents/{path..}`: server-made commits in a bare repository. This is +the first F3 code that writes objects and refs for a client over the API. Recorded exchanges: E#36 to #43 (create, +duplicate, update, stale sha, missing sha, `new_branch` from create and update, duplicate branch), B#27/28 (read-only +token 403, routed already), Q and W (protection, F4). Reviewed by Opus and Sol (see "Decisions after independent +review" at the end, which supersedes anything earlier in this file that it contradicts). + +## Request and response + +- `POST` creates, `PUT` updates, `DELETE` deletes a file. Body JSON: `content` (base64; required for POST and PUT, 422 + when missing or not a string), `message`, `branch`, `new_branch`, `sha` (PUT and DELETE: the blob id being replaced or + removed). `author`, `committer`, `dates` and `signoff` are accepted and **ignored** (the authenticated user is the + identity; departure entry); `from_path` (rename) is refused with 422; other unknown keys are ignored. An empty + `branch` or `new_branch` is absent, and `new_branch` equal to `branch` is absent. +- Success: 201 (POST) / 200 (PUT, DELETE) with `{content, commit, verification}`. `content` is the contents object of + the written file at the new commit (`null` for DELETE), `commit` is the flat file-commit object `{url, sha, created, + html_url, author, committer, parents[{url,sha,created}], message, tree{url,sha,created}}` with the **real** tree id + (not F3a's commit-id quirk), `verification` the unsigned object. The whole response is built in memory before the ref + moves; nothing is read from git afterwards. +- `message`: trailing whitespace trimmed and one `\n` added (recorded `"add new.txt\n"`); empty means `Add|Update|Delete + `; NUL is 422. +- Recorded errors: POST on an existing path 422 `repository file already exists [path: p]`; PUT without `sha` 422 + `[SHA]: Required`; stale `sha` 422 `sha does not match [given: g, expected: e]`; `new_branch` that exists 422 + `branch already exists [name: n]`. Added: missing `branch` 404 `branch does not exist [name: b]`; PUT or DELETE of a + missing path 404 `repository file does not exist [path: p]` (unrecorded; Gitea 1.22 may answer 500); bad base64, + invalid path or branch name, a path blocked by a non-tree parent: 422 naming the path. + +## Semantics + +- `branch` is the base (default: the default branch **from the row read under the lock**). Without `new_branch` the + commit advances `branch`; with it the commit is parented on the base tip and creates `new_branch`, which must be a + valid branch name that neither exists nor collides (equal, a path prefix of, or prefixed by) with an existing branch + name; checked before any object is written (422). +- **Unborn default.** If the base is the default branch and it does not resolve (the F3a/`empty` definition, whatever + other branches exist), POST writes a parentless commit and creates the default branch. This is how a portal seeds an + `empty: true` repository. PUT and DELETE there are 404 `branch does not exist`; `new_branch` from an unborn base is + 404. A missing non-default base is 404. +- Only regular files: new files `100644`; PUT keeps an existing `100755`; PUT/DELETE of a symlink, gitlink or directory + is 422; POST on any existing entry, or under a parent that is a non-tree entry, is 422. +- **Path rules** (before any git call): `valid_path`, no `\`, no trailing `/`, at most 64 components, each at most 255 + bytes, none a `.git` alias (case, trailing dots/spaces, `:stream`, `git~1`, HFS-ignorable code points) and none a + `.gitmodules` alias (same normalisation, `gitmod~1`): **`.gitmodules` cannot be written through the API** (422; + its fsck checks on URL/path/name only run for pushes). `.gitattributes` content is refused when a line exceeds + 2048 bytes (fsck's `gitattributesLineLength`). The guarantee is git-metadata aliases only; Windows device names, + `:` inside a name and case-fold collisions between siblings are not prevented (threat model). +- **Identity.** Author and committer: the authenticated user's `login` and stored email, else + `@noreply.localhost`; dates are one server timestamp passed to git explicitly so the response matches the + commit exactly. +- **Size.** The JSON body limit (`--max-body-bytes`, 1 MiB) bounds the content. Operation limits: at most 64 path + levels, 16 MiB of raw tree listing per level and 32 MiB cumulative, 60 s for the whole operation (checked between + steps), 3 publication attempts. Free disk is checked against the same floor as pushes (`--min-free-bytes`) with a + reservation for the content plus rebuilt trees. + +## Algorithm (object ids and raw bytes only; bare repository, no index file) + +**Lock order, everywhere, once each:** the per-repository **write gate** (timed `Gate(1)`, 503 `try again` when +busy, in the same per-id map as the lifecycle lock so `delete` drops both), then the shared lifecycle lock via a new +`Lifecycle::with_write_row` that returns the fresh `live` row (default branch included). The lifecycle lock is never +re-acquired beneath it and never upgraded; helpers below this boundary use no `read_locked`. Recorded in the +`lifecycle.rs` module doc as the rule for F4. + +1. Validate everything cheap (path, names, base64, message, fields) and check the protection policy once early + (Gitea's order: protection before preconditions). +2. Resolve the base: `rev-parse --verify` of `refs/heads/` to `old`, or unborn per the rules above. For + `new_branch`, list branches and check collisions. +3. Walk the path from `old^{tree}` with raw `ls-tree -z` (no `-l`), one level at a time, parsed as **bytes** + (`mode`, `type`, `oid`, name bytes). Check preconditions on these entries (exists / kind / `sha`). +4. `hash-object -w --no-filters --stdin` for the blob (reused on retries); rebuild directories bottom-up with `mktree -z` + from the same raw entries with the one entry replaced, added or dropped (an emptied directory is dropped from its + parent); names are never decoded. A `mktree` failure (it rejects a gitlink whose id is a blob) is 422 + `directory cannot be rewritten` with nothing published. Modes git itself canonicalises on listing (`100664`) are + normalised in rebuilt directories only; untouched subtrees keep their ids. +5. `commit-tree [-p old]`, identity and dates in the environment, message on stdin. +6. **Protection seam:** `policy::check_update`'s sibling for contents (`check_contents_write(target, creating)`) is + evaluated for the *target* ref on every publication attempt, including `new_branch`; today it permits. The `.sc` + hook path and this one share the module so F4 adds a rule once. +7. `update-ref refs/heads/ ` (`expected` is `old`, or the zero id to create). **Outcome table** + (re-read of the target ref after any failure; a re-read that itself fails is 500, never "missing"; every failure is + logged with stderr, and a message naming a `.lock` file gets an operator hint): + - ref now equals `new`, or has `new` as an ancestor: success (a killed process whose update landed, even if a + push has built on it since); + - advance, ref gone: 404 `branch does not exist`; + - advance, ref changed: lost round, back to step 2 for the new tip (preconditions re-evaluated: the PUT/DELETE + `sha` must still match, a POST path must still be absent; the blob is reused); + - advance, ref unchanged and stderr names a held lock: lost round after 50 ms; + - create of `new_branch`, ref exists: 422 `branch already exists`; + - bootstrap create of the default branch, ref exists: lost round (advance on the new tip); + - create blocked by a ref D/F conflict that raced the early check: 422 `branch already exists`; + - anything else: 500. + After 3 rounds: 409 `the branch was changed concurrently, try again` (no "conflict" in the text: clients read it as a + merge conflict). +8. Log-only after publication: `repos::touch`. Respond from the prebuilt values. + +Crash safety: objects first, ref last; failures and lost rounds leave unreferenced objects (no automatic gc runs; +`gc` is F5). No temporary files exist. + +## Tests + +Unit (`gitwrite`): path rules (git's t0060 style vectors for `.git`, `.gitmodules` and aliases, depth, length), raw +tree parse/rebuild round trip with non-UTF-8, tab and newline names and a legacy mode, message normalisation, +`.gitattributes` line limit, the publish loop against a fake ref store (lost rounds, exhaustion, killed-but-landed, +gone, locked). +E2E (`e2e_write.rs`, real git reads the result back): create/update/delete round trips visible to `git clone` and the F3a +reads; nested create; delete prunes emptied directories; `100755` kept; symlink, gitlink and directory refused; D/F both +ways; stale, missing sha; `new_branch` exists / is a prefix of / is under an existing branch; an untouched sibling with a +non-UTF-8 name and one with tab+newline keep their exact bytes; first commit in an empty repository; unborn default +with another branch present; PUT on unborn 404; read-only token 403; non-collaborator 404; an independent real git +ref writer racing API writes (assert ancestry and both files after a retried success, or the 409); the exhaustion +case publishes nothing from the API; a response never depends on a git call after the ref update; `fsck --strict` +clean; busy runner 503 with no ref change; a queued repository delete does not deadlock with a write; the +response `commit.tree` matches `git cat-file`. +Conformance: remove E's stop and defers (E#36 to #43) and the POST/PUT 501 stubs; DELETE gets an implementation with +no recorded exchange (unrecorded in the spec). Spec text for "Contents write" is updated in the same PR (`mktree`, not +a temporary index; `.gitmodules`; `noreply.localhost`). + +## Decisions after independent review + +Opus and Sol both reviewed this plan (first version used `gitobj`'s lossy listing and a mutex-then-lock order). +Accepted: byte-preserving tree handling (B1, Sol 1); full `update-ref` outcome table and branch-collision pre-check +(B2); depth/size/time limits and disk floor (B3, Sol 3); the unborn-default rule above (B4; Opus's "base is the unborn +default" over Sol's "genuinely empty" because sc-forge itself leaves `empty: true` repositories with other branches +after a push); lock order and one fresh row under the guard (Sol 4, 5, Opus discussion 1); response built before the +ref moves, touch log-only (Sol 6, Opus 2); `.gitmodules` refused and `.gitattributes` line limit (Sol 2, Opus 3); +`new_branch` equal to `branch` is absent; stale-lock hint; protection checked for the target on every attempt including +`new_branch`, and early in Gitea's order. +Disagreement: `author`/`committer`/`dates`. Sol: refuse with 422. Opus: accept and ignore, since refusing known Gitea +fields breaks clients that send them while the identity is the authenticated user either way. Decision: ignore, with a +`departures.json`-style note in the spec; `from_path` is refused because ignoring a rename leaves a copy. +Not done: portable-name collision rules (documented limit); sharing the push disk reservation beyond the free-space +floor; a temporary-index route (rejected: it also canonicalises modes and costs a file per request). diff --git a/docs/superpowers/specs/2026-10-22-p38-forge-design.md b/docs/superpowers/specs/2026-10-22-p38-forge-design.md index 5be1fe74..4353277d 100644 --- a/docs/superpowers/specs/2026-10-22-p38-forge-design.md +++ b/docs/superpowers/specs/2026-10-22-p38-forge-design.md @@ -355,17 +355,16 @@ Minimum `git` 2.40 in tests; the container pins a current release treated as literals with NUL-delimited output (`-z`). Output is size-capped. - **Reads:** `for-each-ref`, `rev-parse`, `log`/`rev-list` with a fixed `-z` format, `cat-file`, `ls-tree -r -z`, `diff-tree`, `archive --format=zip|tar.gz`. -- **Contents write** (POST/PUT): with `new_branch`, the protection policy applies to the +- **Contents write** (POST/PUT/DELETE): with `new_branch`, the protection policy applies to the **target** branch (a new branch has no rule), creating it from a protected base is allowed (recorded: 201 with `new_branch` where a write to the protected branch is - 403), and the base ref is read once and used as the CAS old value. Validate the path (reject empty, `.`, `..`, - `.git` and its NTFS/HFS aliases, absolute paths; mode 100644 only), `hash-object - -w`, a temp `GIT_INDEX_FILE` (`read-tree`, `update-index --cacheinfo`, - `write-tree`), `commit-tree`, then `update-ref ` (zero old id - for an unborn branch). The branch must pass the protection policy first. A - lost CAS rebuilds from the new tip, revalidates the supplied blob `sha`, and - retries up to 3 times, then answers 409. Server-made commits bypass - `receive.fsckObjects`, so the path rules above are the check. + 403), and the base ref is read once and used as the CAS old value. As built (F3b, "F3b decisions" + below): path rules, then raw `ls-tree -z` and `mktree -z` along the path (no index file), `hash-object -w`, + `commit-tree`, then `update-ref ` (zero old id to create). The branch must pass the + protection policy first and again immediately before the ref update. A lost CAS rebuilds from the new tip, + revalidates the supplied blob `sha`, and retries up to 3 times, then answers 409. Server-made commits bypass + `receive.fsckObjects`, so the path rules above are the check (plus a refusal of `.gitmodules` and over-long + `.gitattributes` lines, whose content `fsck` checks on pushes). - **Per-repo write lock** serializes the server's own ref writes (contents, merges, PR-ref updates). It does not exclude receive-pack; the guarantees below come from Git's atomic ref transaction, not the lock. @@ -816,8 +815,7 @@ F2b is the git half of F2 (plan: `docs/superpowers/plans/2026-10-23-p38-f2b-git- ### F3a decisions (as built) F3a is the read half of the content API (plan: `docs/superpowers/plans/2026-10-23-p38-f3a-content-reads.md`). The -write half (`POST`/`PUT` contents) is F3b; until then both answer 501 after the auth checks, so a read-only token still -gets its recorded 403. +write half (`POST`/`PUT`/`DELETE` contents) is F3b, below; F3a shipped them as 501 stubs behind the auth checks. - **Routes.** `git/refs`, `git/refs/{ref..}`, `tags`, `commits`, `git/commits/{sha}` (+ `.diff`/`.patch`), `git/trees/{sha..}`, `contents`, `contents/{path..}`, `archive/{name..}` under `/repos/{o}/{r}` (read scope, read @@ -858,6 +856,51 @@ gets its recorded 403. contents write (`[42, KeyError: 'sha']`) with E#36 to #41 deferred to F3b; `D` defers its PR listing and `B` its status post (F4). +### F3b decisions (as built) + +F3b is the write half of the content API (plan: `docs/superpowers/plans/2026-10-24-p38-f3b-contents-writes.md`, with its +"Decisions after independent review"). `gate E` replays E#36 to #43 unchanged. + +- **Routes.** `POST`, `PUT` and `DELETE` `/repos/{o}/{r}/contents/{path..}` (write scope, write permission). `DELETE` is + unrecorded (the recordings only have POST and PUT); it follows the Gitea 1.22 API: `sha` required, `{content: null, + commit, verification}`. +- **Request.** `content` base64 (required for POST/PUT, an empty string is an empty file), `message` (trailing + whitespace trimmed, one `\n` added, default `Add|Update|Delete `, NUL refused), `branch` (base, default the + default branch from the row read under the lock), `new_branch` (empty or equal to `branch` is absent), `sha` (PUT and + DELETE). `author`, `committer`, `dates` and `signoff` are accepted and ignored (the authenticated user is author and + committer; departure from Gitea); `from_path` is 422 because ignoring a rename would leave a copy. +- **Path rules** (`gitwrite::write_path_ok`): `valid_path`, no `\`, no trailing `/`, at most 64 components of at most 255 + bytes, no `.git` alias (case, trailing dots and spaces, `:stream`, `git~1`, HFS-ignorable code points) and no + `.gitmodules` alias: **`.gitmodules` cannot be written through the API** (its URL/path/name checks run only in + `receive.fsckObjects`). A `.gitattributes` line of 2048 bytes or more is refused, under every NTFS/HFS alias of the name; `.gitmodules` is refused under every alias including the `gitmod~N` and `gi7eba~N` short names. Windows device names, `:` in a name and + case-fold sibling collisions are not prevented. +- **Semantics.** New files are `100644`; PUT keeps `100755` (and normalises legacy modes such as `100664` in the directories + it rebuilds; untouched subtrees keep their ids); a symlink, gitlink or directory at the path is 422 for PUT/DELETE, and + POST on any existing entry or under a non-tree parent is 422. An unborn default branch (`empty: true`, whatever other + branches exist) takes a POST as its first, parentless commit; PUT/DELETE there and `new_branch` from an unborn base are + 404. `new_branch` must not equal, contain or lie under an existing branch name (422, before any object is written). + A missing path on PUT/DELETE is 404 (unrecorded; Gitea may answer 500). +- **Locks.** A per-repository write gate (`limits::Gate(1)`, 10 s, 503 when busy) is taken first, then the shared + lifecycle lock, each once (`Lifecycle::with_write_row`, which hands over the fresh row); documented in + `lifecycle.rs` as the order F4 must follow. The runner gate applies to every git command. +- **Publication.** Objects first, the ref last by `update-ref` CAS. After any failure the ref is re-read and the outcome + decided by `gitwrite::classify` (landed, which includes a tip that has our commit as an ancestor, as when a push built on a killed `update-ref` that had succeeded; lost round, lock held with a 50 ms pause, branch gone 404, `new_branch` exists + 422, ref D/F conflict 422, anything else 500, logged); three lost rounds is 409 `the branch was changed concurrently, + try again`, or 503 `the branch is locked` when every round met a held ref lock (a killed git can leave a stale + `refs/heads/.lock`; the log then tells the operator to remove it). The response is built before the ref moves and nothing is read from git afterwards; the repository's + `updated_unix` stamp after a successful write is log-only on failure. +- **Limits.** Body (`--max-body-bytes`, 1 MiB) bounds content; 64 levels, 16 MiB per raw listing and 32 MiB cumulative + listing per attempt, 60 s for the operation (checked between attempts and between the walk, the rewrite and the commit; one git command may still run up to its own limit), free disk against `--min-free-bytes` through the same reservation pool as + pushes (content plus 8 MiB, and per attempt the tree bytes it read). +- **Protection seam.** `policy::check_contents_write(target, creating)` is evaluated early (Gitea checks protection first) + and again immediately before the ref update on every attempt, including for `new_branch`; it permits everything until + F4 adds rules, in the same module as the pre-receive policy. +- **Identity.** Author and committer are the authenticated user's login and stored email (else + `@noreply.localhost`); both commit dates are one server timestamp (`GIT_*_DATE`) so the response matches the commit. +- **Not built / not tested.** The 409 exhaustion path has a unit-tested classifier but no end-to-end test (it needs a + deterministic interfering writer); the F4 `refs/forge/pull//head` refresh for a contents write to a PR branch; + portable-name collision rules; a disk reservation sized to the exact rebuilt trees. + ## Open questions 1. The merge of a PR whose head is already in base (recorded: 405 `try again later`,