diff --git a/.agents/skills/layerfs-js-sdk/SKILL.md b/.agents/skills/layerfs-js-sdk/SKILL.md index 9ceeb7bfac..86cdc40c9c 100644 --- a/.agents/skills/layerfs-js-sdk/SKILL.md +++ b/.agents/skills/layerfs-js-sdk/SKILL.md @@ -295,7 +295,7 @@ and examples. | [commit](references/methods/commit.md) | `list(project, branch)`, `get(project, branch, commit)`, `diff(project, from_commit, to_commit, { paths_only, … })` | | [file](references/methods/file.md) | reads `read`, `stat`, `list`, `glob`, `grep`, `blame`, `diff`; writes `write`, `edit`, `rm`, `mkdir`, `mv`, `cp`, `apply` | | [sandbox](references/methods/sandbox.md) | `create(project, name, machine?, { image })`, `list(project?, { page_size, page_pos })`, `get(name, { page_size, page_pos })`, `close(name, { force })` | -| [workspace](references/methods/workspace.md) | `mount(sandbox, branch, { name })`, `exec(ws, command, { message, stream, stdout, stderr })`, `commit(ws, { message, force })`, `unmount(ws, { discard, force })`, `status(ws)` | +| [workspace](references/methods/workspace.md) | `mount(sandbox, branch, { name })`, `exec(ws, command, { message, stream, stdout, stderr })`, `commit(ws, { message, force })`, `unmount(ws, { discard, force, kill })`, `status(ws)` | | [machine](references/methods/machine.md) | `list`, `platforms`, `get`, `status`, `start`, `stop`, `create`, `connect`, `remove`, `disconnect`, `update` | | [store](references/methods/store.md) | `list`, `get`, `status`, `create`, `connect`, `push`, `pull`, `remove`, `disconnect` | | [backup](references/methods/backup.md) | `connect`, `disconnect`, `list`, `create`, `restore` | diff --git a/.agents/skills/layerfs-js-sdk/references/methods/workspace.md b/.agents/skills/layerfs-js-sdk/references/methods/workspace.md index bafb3fd72f..1548a92c99 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/workspace.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/workspace.md @@ -11,7 +11,7 @@ workspace.mount(sandbox, branch, { name }) workspace.exec(ws, command, { message, stream, stdout, stderr }) workspace.commit(ws, { message, force }) -workspace.unmount(ws, { discard, force }) +workspace.unmount(ws, { discard, force, kill }) workspace.status(ws) ``` @@ -157,6 +157,15 @@ const run = await layerfs.workspace.exec(ws, `printf '%s' ${b64} | base64 -d > s too; mount again elsewhere), `NotMounted`, `Unknown` (the exit code is not known: the command may or may not have run; stop). +A foreground command leaves a private `/tmp/layerfs-exec-.record` in its +Sandbox while it runs, naming its Workspace, its pid and that pid's start +time; the record is removed when the command ends. A second client can use it +([`workspace.unmount`](#workspaceunmount) `kill: "workspace"`) when the first +client died before its unmount, leaving the command as an orphan that holds +the mount. The record is written only where the Sandbox has `/proc` (Docker's +container and Linux namespaces; a macOS host-process Sandbox writes none), +and the image must provide util-linux `setsid --wait` (Debian 12 has it). + ## `workspace.commit` `workspace.commit(ws, { message, force })` @@ -190,25 +199,59 @@ leaves their text out. ## `workspace.unmount` -`workspace.unmount(ws, { discard, force })` +`workspace.unmount(ws, { discard, force, kill })` Ends a Workspace. The Sandbox keeps running with no Workspace until `sandbox.close` (#535 D11). | Option | Meaning | | --- | --- | | `discard` | Drop uncommitted changes instead of refusing | -| `force` | Abort the Workspace's kernel connection; drops uncommitted changes. A normal unmount that stalls ends itself with one abort write; use `force` only for a live connection, never to repeat an attempted abort | +| `force` | Abort the Workspace's kernel connection; drops uncommitted changes. A normal unmount that stalls ends itself with one abort write; use `force` only for a live connection, never to repeat an attempted abort. While a foreground command's record names a live process, `force` is refused `Busy`: cancel that command, then unmount normally, so it cannot create a new `Retained(Detach)` | +| `kill` | `"workspace"` (the only accepted value) to cancel this Workspace's own foreground commands first, then unmount normally: TERM to each process group, a bounded wait, then KILL only if still there. It never stops or deletes the Sandbox, never aborts the Workspace's connection, and never touches another Workspace. Any other value is `InvalidInput`, as is combining it with `force` | **Returns:** `{ kind: "Unmounted" }`, or with `force` -`{ kind: "ForceUnmounted", abort, detach, cleanup }`. +`{ kind: "ForceUnmounted", abort, detach, cleanup }`. A `kill: "workspace"` +unmount that ends the Workspace answers `{ kind: "Unmounted" }`: it is a +normal unmount after the holders are cancelled, not a forced one. | Code | Meaning | Do | | --- | --- | --- | | `Dirty` | Uncommitted changes; `detail.changed` lists paths, with `pendingMerge` and `truncated` | Commit them so the step is recorded; discard only a Workspace you're abandoning | -| `Busy` | A process still runs there (`detail.processes`) | Wait for it, or end it in a command | +| `Busy` | A foreground command still runs there; `detail.processes` lists the ids | `workspace.unmount(ws, { kill: "workspace" })` cancels only this Workspace's holders and unmounts; a holder that survives makes it `Busy` again with the ids | +| `InvalidInput` | `kill` was not `"workspace"`, or `kill` and `force` were combined | Pass only `kill: "workspace"`, without `force` | | `NotMounted` | Already unmounted, or the wrong `ws` | Use the Workspace from `mount` | | `Failed` with `TeardownCustody` in the message | The mount is gone but its drain did not finish | Preserve the message; don't repeat the abort; replace the Sandbox if its connection can't be ended | +**Example** + +```js +await layerfs.workspace.unmount(ws); // ws is now invalid +``` + +After a `Busy` refusal, cancel only this Workspace's holders and unmount again: + +```js +try { + await layerfs.workspace.unmount(ws); +} catch (error) { + if (error.code !== "Busy") throw error; + await layerfs.workspace.unmount(ws, { kill: "workspace" }); +} +``` + +`kill: "workspace"` is the recovery for a Workspace whose own foreground +command was orphaned by a killed client. It cancels only the records whose +Workspace path is this one, and a process group with a member whose cwd or +open file is outside the Workspace is left alone: the call answers `Busy` +with `detail.scope_conflict` instead of taking collateral. It is not a +whole-Sandbox stop: `sandbox.close` has no force mode, and a Workspace the +daemon already holds as `Retained(Detach)` needs its custody retry; `kill` +cannot make that Workspace unmountable. A process that daemonized and escaped +the exec's session has no record once the command ends, so it needs a +separate explicit destroy. The cancel runs where the Sandbox's runtime is (its +machine's worker, or this process), so a Sandbox this process reaches only +through a forwarded daemon port is handled the same way. + ## `workspace.status` `workspace.status(ws)` diff --git a/api/crates/layerfs-api/src/workspace/unmount.rs b/api/crates/layerfs-api/src/workspace/unmount.rs index 1700ed259d..7dc4bb55a6 100644 --- a/api/crates/layerfs-api/src/workspace/unmount.rs +++ b/api/crates/layerfs-api/src/workspace/unmount.rs @@ -10,7 +10,7 @@ pub(crate) const CALL: Call = call( "workspace", "unmount", &[a("ws", Text)], - &[opt("discard", Bool), opt("force", Bool)], + &[opt("discard", Bool), opt("force", Bool), opt("kill", Text)], "end a Workspace", run, ); diff --git a/api/crates/layerfs-api/tests/workspace_names.rs b/api/crates/layerfs-api/tests/workspace_names.rs index ffa8cba975..0e0c4738f2 100644 --- a/api/crates/layerfs-api/tests/workspace_names.rs +++ b/api/crates/layerfs-api/tests/workspace_names.rs @@ -41,15 +41,20 @@ fn home_with_a_named_workspace(tag: &str) -> PathBuf { &SdkValue::map([("name", t("app")), ("created", SdkValue::Int(1))]), ) .unwrap(); - // the native daemon a call would launch: never started here + // the native daemon a call would launch: never started here, but the + // record must name a file so the call reaches the Sandbox itself let native = paths::machine_record(&home, "native"); // (a home made without the machine calls has no native record yet) let record = read_record(&native) .unwrap() .unwrap_or_else(|| SdkValue::map([("name", t("native")), ("platform", t("native"))])); + // the recorded daemon must exist for a call to reach the Sandbox; this + // placeholder is never launched, because no daemon keys are recorded + let daemon = home.join("layerfs-daemon"); + std::fs::write(&daemon, b"").unwrap(); write_record( &native, - &home::with_field(record, "daemon", t("/nonexistent")), + &home::with_field(record, "daemon", t(&daemon.display().to_string())), ) .unwrap(); write_record( diff --git a/api/crates/layerfs-control/src/sandbox/dispatch.rs b/api/crates/layerfs-control/src/sandbox/dispatch.rs index bc2f37d2d0..bf698b4c08 100644 --- a/api/crates/layerfs-control/src/sandbox/dispatch.rs +++ b/api/crates/layerfs-control/src/sandbox/dispatch.rs @@ -31,6 +31,7 @@ pub fn call(op: &str, args: SdkValue) -> Outcome { "workspace.commit" => workspace::commit(&workspace::by_name(args)?), "workspace.unmount" => workspace::unmount(&workspace::by_name(args)?), "workspace.exec" => exec::exec(&workspace::by_name(args)?), + "workspace.holders" => exec::holders(&workspace::by_name(args)?), _ => Err(refused("UnknownMethod", format!("no operation {op}"))), } } diff --git a/api/crates/layerfs-control/src/sandbox/exec.rs b/api/crates/layerfs-control/src/sandbox/exec.rs index 61d32fed37..b18d83241f 100644 --- a/api/crates/layerfs-control/src/sandbox/exec.rs +++ b/api/crates/layerfs-control/src/sandbox/exec.rs @@ -1,38 +1,33 @@ //! Commands in a mounted Workspace, run by the Sandbox runtime owner. //! -//! `exec` runs `bash -c ` as the Sandbox's command identity in the +//! `exec` runs the command as the Sandbox's command identity in the //! Workspace directory and returns the exit code and output when it ends; the //! daemon is not involved. A local Sandbox //! (`layerfs_sandbox::Sandbox::HostProcess`) runs the same commands as //! processes of this kernel. +//! +//! A foreground command writes a private record in the Sandbox's `/tmp` +//! naming its Workspace, its pid and that pid's start time before it runs, so +//! a later call can find and stop an orphan whose first client is gone. The +//! executor owns command launch and cancellation (AGENTS.md), so the record +//! is written where the command runs, by the same runtime owner: `bash -c` +//! under util-linux `setsid --wait`, and `/proc` read inside the Sandbox. A +//! backend without `/proc` (the macOS host process) runs the command plainly +//! and leaves no record; nothing else changes. +//! +//! This module is the orchestration: the two entrypoints and the result they +//! shape. [`record`] owns the foreground record and the id it carries, +//! [`holders`] finds and stops the processes a record names, and [`runtime`] +//! adapts this Sandbox's runtime to the run and capture calls they make. use crate::common::outcome::Outcome; -use crate::sandbox::output::Spool; -use crate::sandbox::registry::{sandbox, text, SandboxState}; -use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use crate::sandbox::registry::{sandbox, text}; +use layerfs_bridge::control::SdkValue; -/// One command to its end: exit code, stdout, stderr. `spool` sends every -/// non-empty stream to its file instead of keeping it inline. Its home is -/// the Sandbox's scratch directory (`/tmp` in a container), since the -/// command identity has no account there. -fn run( - state: &SandboxState, - directory: &str, - arguments: Vec, - spool: bool, -) -> Result<(i32, Spool, Spool), SdkRefusal> { - let runtime = state.runtime()?; - crate::sandbox::runtime::run( - runtime, - state.identity, - arguments, - vec![format!( - "HOME={}", - crate::sandbox::runtime::scratch(runtime) - )], - directory, - spool, - ) -} +mod holders; +mod record; +mod runtime; + +pub(crate) use holders::busy; /// `workspace.exec(ws, command, { spool? })`. /// @@ -53,12 +48,9 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { Some(directory) => directory.to_owned(), None => state.directory(path)?, }; - let (code, stdout, stderr) = run( - &state, - &directory, - vec!["bash".into(), "-c".into(), command.into()], - args.get_bool("spool"), - )?; + let arguments = record::foreground_arguments(&state, command)?; + let (code, stdout, stderr) = + runtime::run(&state, &directory, arguments, args.get_bool("spool"))?; let mut streams = crate::sandbox::output::fields(stdout, stderr)?.into_iter(); let mut result = vec![ ("kind", SdkValue::Text(format!("exit:{code}"))), @@ -75,3 +67,30 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { .collect(), )) } + +/// `workspace.holders(ws)`: the foreground execs still running in a +/// Workspace, read, or cancelled first with `kill`. This op runs where the +/// Sandbox's runtime is (its machine's worker, or this process), so a caller +/// that reaches only its daemon's forwarded port can still cancel an orphan. +/// Answers `{ processes }`; with `kill`, each own process group gets TERM and +/// then KILL, and a group that reaches outside the Workspace answers `Busy`. +pub(crate) fn holders(args: &SdkValue) -> Outcome { + let state = sandbox(args)?; + let path = text(args, "path")?; + if args.get_bool("kill") { + holders::cancel(&state, path)?; + } + let records = record::foreground_in(&state, path)?; + Ok(SdkValue::map([ + ("path", SdkValue::text(path)), + ( + "processes", + SdkValue::List( + records + .into_iter() + .map(|record| SdkValue::Text(record.id)) + .collect(), + ), + ), + ])) +} diff --git a/api/crates/layerfs-control/src/sandbox/exec/holders.rs b/api/crates/layerfs-control/src/sandbox/exec/holders.rs new file mode 100644 index 0000000000..9d2ae447b3 --- /dev/null +++ b/api/crates/layerfs-control/src/sandbox/exec/holders.rs @@ -0,0 +1,256 @@ +//! Finding and stopping the holders of one Workspace: the same-group `/proc` +//! scan, the scope probe, TERM then KILL, and the bounded waits between them. +use super::record::{self, Foreground}; +use super::runtime; +use crate::common::outcome::refused; +use crate::sandbox::registry::SandboxState; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use std::time::Duration; + +/// How many times a stopped process group is looked for. +const GONE_CHECKS: u32 = 50; +/// Pause between those checks. +const GONE_PAUSE: Duration = Duration::from_millis(100); +/// The cwd and open files of every member of process group `$1`, when any is +/// outside `$2`: one `pid path` line each, read from `/proc`. Outside means an +/// absolute path that is not the Workspace or below it, not `/dev`, and not a +/// framework file under `/tmp/layerfs-` (the exec's own record and log); a +/// pipe, socket or deleted-or-anonymous file has no path and is not a +/// conflict. +const GROUP_OUTSIDE: &str = r#"pgid=$1 +root=$2 +outside() { + case "$1" in + "$root"|"$root"/*) return 1 ;; + /dev|/dev/*) return 1 ;; + /tmp/layerfs-*) return 1 ;; + /*) return 0 ;; + *) return 1 ;; + esac +} +for f in /proc/[0-9]*/stat; do + s=$(cat "$f" 2>/dev/null) || continue + set -- ${s##*) } + [ "$3" = "$pgid" ] || continue + pid=${f#/proc/} + pid=${pid%/stat} + c=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue + if outside "$c"; then printf "%s %s\n" "$pid" "$c"; continue; fi + for l in /proc/$pid/fd/*; do + t=$(readlink "$l" 2>/dev/null) || continue + if outside "$t"; then printf "%s %s\n" "$pid" "$t"; break; fi + done +done +exit 0"#; + +/// What a workspace-scoped kill made of one holder. +#[derive(Debug, PartialEq, Eq)] +enum Terminated { + /// The holder is gone (or already was). + Gone, + /// Signals were sent but the process group is still there. + Survived, + /// The process group has a member whose cwd or open file is outside the + /// target Workspace: nothing was signalled. Each entry is `pid path`. + Outside(Vec), +} + +/// A `Busy` refusal naming the foreground execs that still run in the +/// Workspace. +pub(crate) fn busy(sandbox: &str, path: &str, processes: Vec) -> SdkRefusal { + SdkRefusal { + code: "Busy".to_owned(), + message: format!( + "{sandbox}:{path} is busy: {} still running", + processes.join(", ") + ), + detail: SdkValue::map([( + "processes", + SdkValue::List(processes.into_iter().map(SdkValue::Text).collect()), + )]), + } +} + +/// A `Busy` refusal naming holders whose process group reaches outside the +/// Workspace, so a workspace-scoped kill left them alone. +fn scope_conflict( + state: &SandboxState, + path: &str, + conflicts: Vec<(String, Vec)>, +) -> SdkRefusal { + let ids = conflicts + .iter() + .map(|(id, _)| SdkValue::text(id.as_str())) + .collect::>(); + let outside = conflicts + .iter() + .flat_map(|(_, paths)| paths.iter()) + .map(|path| SdkValue::text(path.as_str())) + .collect::>(); + SdkRefusal { + code: "Busy".to_owned(), + message: format!( + "{}:{path} has a holder whose process group reaches outside the Workspace; nothing was killed", + state.id + ), + detail: SdkValue::map([ + ("processes", SdkValue::List(ids)), + ("scope_conflict", SdkValue::List(outside)), + ]), + } +} + +/// Cancels every foreground exec still running in the Workspace at `path`: +/// TERM to its process group, a bounded wait, then KILL only if it is still +/// there. A group that reaches outside the Workspace is never signalled; a +/// holder that survives answers `Busy` with its id. +pub(super) fn cancel(state: &SandboxState, path: &str) -> Result<(), SdkRefusal> { + let records = record::foreground_in(state, path)?; + if records.is_empty() { + return Ok(()); + } + let mut conflicts = Vec::new(); + for record in &records { + if let Terminated::Outside(outside) = terminate(state, path, record)? { + conflicts.push((record.id.clone(), outside)); + } + } + // Never kill a group that reaches outside this Workspace: leave it and + // name it, so the caller decides. No silent collateral. + if !conflicts.is_empty() { + return Err(scope_conflict(state, path, conflicts)); + } + let remaining = record::foreground_in(state, path)?; + if !remaining.is_empty() { + return Err(busy( + &state.id, + path, + remaining.into_iter().map(|record| record.id).collect(), + )); + } + Ok(()) +} + +/// Stops one holder of the Workspace at `path` for a workspace-scoped kill: +/// TERM to its process group, a bounded wait, and KILL only if it is still +/// there, then a second bounded wait. A group with a member outside the +/// Workspace is left alone. A foreground exec's wrapper removes its `/tmp` +/// record once that session ends. +fn terminate( + state: &SandboxState, + path: &str, + record: &Foreground, +) -> Result { + if !owned_alive(state, record.pid, &record.start)? { + return Ok(Terminated::Gone); + } + let outside = outside_scope(state, record.pid, path)?; + if !outside.is_empty() { + return Ok(Terminated::Outside(outside)); + } + signal(state, record.pid, "TERM")?; + let mut gone = wait_gone(state, record.pid)?; + if !gone { + signal(state, record.pid, "KILL")?; + gone = wait_gone(state, record.pid)?; + } + Ok(if gone { + Terminated::Gone + } else { + Terminated::Survived + }) +} + +/// The members of process group `pid` whose cwd or open file is outside the +/// Workspace at `path`, as `pid path` lines. +fn outside_scope(state: &SandboxState, pid: u32, path: &str) -> Result, SdkRefusal> { + let (code, out) = runtime::capture( + state, + "/tmp", + vec![ + "bash".into(), + "-c".into(), + GROUP_OUTSIDE.into(), + "layerfs-kill".into(), + pid.to_string(), + path.into(), + ], + )?; + if code != 0 { + return Err(refused( + "Failed", + format!("process-group scope probe exited {code}"), + )); + } + Ok(out + .lines() + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect()) +} + +/// The start time of `pid`, as `/proc` names it. +fn start_time(state: &SandboxState, pid: u32) -> Result, SdkRefusal> { + let (code, out) = runtime::capture( + state, + "/tmp", + vec![ + "bash".into(), + "-c".into(), + format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\""), + ], + )?; + Ok((code == 0).then_some(out)) +} + +/// True while the process group of `pid` has a member that is not a zombie. +/// The container's first process is the daemon, which does not reap orphans, +/// so a finished member can stay a zombie that `kill -0` still finds. +fn alive(state: &SandboxState, pid: u32) -> Result { + // After the `(comm) ` field: state, parent, process group. + let probe = format!( + "for f in /proc/[0-9]*/stat; do s=$(cat \"$f\" 2>/dev/null) || continue; \ + set -- ${{s##*) }}; [ \"$1\" != Z ] && [ \"$3\" = {pid} ] && exit 0; done; exit 1" + ); + let (code, _) = runtime::capture(state, "/tmp", vec!["bash".into(), "-c".into(), probe])?; + Ok(code == 0) +} + +fn owned_alive(state: &SandboxState, pid: u32, start: &str) -> Result { + if !alive(state, pid)? { + return Ok(false); + } + if start_time(state, pid)?.as_deref() != Some(start) { + return Err(refused( + "Unknown", + "process group identity is no longer provable; no signal sent", + )); + } + Ok(true) +} + +/// One signal to the process group `pid` leads. The command's own failure is +/// ignored: a group that is already gone is the outcome we want. +fn signal(state: &SandboxState, pid: u32, name: &str) -> Result<(), SdkRefusal> { + runtime::capture( + state, + "/tmp", + vec![ + "bash".into(), + "-c".into(), + format!("kill -{name} -- -{pid} 2>/dev/null; true"), + ], + )?; + Ok(()) +} + +/// Waits (bounded) until the process group `pid` leads has no live member. +fn wait_gone(state: &SandboxState, pid: u32) -> Result { + for _ in 0..GONE_CHECKS { + if !alive(state, pid)? { + return Ok(true); + } + std::thread::sleep(GONE_PAUSE); + } + Ok(false) +} diff --git a/api/crates/layerfs-control/src/sandbox/exec/record.rs b/api/crates/layerfs-control/src/sandbox/exec/record.rs new file mode 100644 index 0000000000..c5069d9e1a --- /dev/null +++ b/api/crates/layerfs-control/src/sandbox/exec/record.rs @@ -0,0 +1,180 @@ +//! The foreground record a running command leaves in the Sandbox's `/tmp`: +//! the wrapper that creates and removes it, the id it carries, and the scan +//! that reads it back. Only a backend with `/proc` writes one. +use super::runtime; +use crate::common::outcome::refused; +use crate::sandbox::registry::{random, SandboxState}; +use layerfs_bridge::control::SdkRefusal; +use layerfs_sandbox::Backend; + +/// Hex digits `process_token` writes: 12 random bytes, two digits each. One +/// width only: a narrower token is not an id this binding generates. +const PROCESS_TOKEN_HEX: usize = 24; +/// Runs one foreground command in a new session and leaves a record in +/// `/tmp` while it runs. argv: `bash -c FOREGROUND_WRAPPER layerfs-exec +/// `; `$0` is the marker, `$1` the id and `$2` the command, passed +/// as its own argv and never concatenated into a script. The inner shell +/// writes the Workspace, its own pid and that pid's start time before it +/// `exec`s the command; the outer shell removes the record when the new +/// session ends. `setsid --wait` is util-linux (Debian 12 has it). +const FOREGROUND_WRAPPER: &str = r#"id=$1 +cmd=$2 +setsid --wait bash -c 'id=$1 +cmd=$2 +p=$$ +s=$(cat /proc/$$/stat 2>/dev/null) || exit 125 +set -- ${s##*) } +shift 19 +start=$1 +umask 077 +r=/tmp/layerfs-exec-$id.record +t=$r.$$ +printf "%s %s %s %s\n" "$id" "$PWD" "$p" "$start" > "$t" || { rm -f "$t"; exit 125; } +mv -f "$t" "$r" || { rm -f "$t"; exit 125; } +exec bash -c "$cmd"' _ "$id" "$cmd" +s=$? +rm -f "/tmp/layerfs-exec-$id.record" +exit $s +"#; + +/// Prints one line per live foreground record: id, Workspace, pid, start. +/// A record is live while its pid exists, is not a zombie, and still has the +/// start time the record names. Every record in `/tmp` is scanned; the caller +/// keeps only its own Sandbox's. +const FOREGROUND_SCAN: &str = r#"for f in /tmp/layerfs-exec-proc_*.record; do +[ -f "$f" ] || continue +read -r id path pid start < "$f" || continue +[ -n "$id" ] && [ -n "$path" ] && [ -n "$start" ] || continue +case "$pid" in ''|*[!0-9]*) continue ;; esac +s=$(cat "/proc/$pid/stat" 2>/dev/null) || continue +set -- ${s##*) } +[ "$1" != Z ] || continue +shift 19 +[ "$1" = "$start" ] || continue +printf "%s %s %s %s\n" "$id" "$path" "$pid" "$start" +done"#; + +/// One live foreground exec, as its `/tmp` record and its process name it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Foreground { + pub(super) id: String, + pub(super) path: String, + pub(super) pid: u32, + pub(super) start: String, +} + +/// The random half of a process id: 12 bytes as 24 lowercase hex digits. +/// `process_id` and the record parser share this one width, so a record the +/// wrapper writes is always one `parse_foreground` accepts. +fn process_token() -> Result { + Ok(layerfs_bridge::sdk_values::hex(&random()?[..12])) +} + +/// The id one exec uses: `proc__<24 hex>`, so a record names the +/// Sandbox whose Workspace it holds. +fn process_id(state: &SandboxState) -> Result { + Ok(format!("proc_{}_{}", state.id, process_token()?)) +} + +/// Whether this Sandbox's backend has `/proc`, so a record can be written and +/// read there: Docker's container is Linux, a namespace Sandbox runs on this +/// Linux kernel, and a macOS host process has neither. +fn records(state: &SandboxState) -> Result { + Ok(matches!( + state.runtime()?.backend(), + Backend::Docker | Backend::Namespaces + )) +} + +/// The argv of one foreground exec: the recording wrapper when the Sandbox +/// has `/proc`, the plain command otherwise. +pub(super) fn foreground_arguments( + state: &SandboxState, + command: &str, +) -> Result, SdkRefusal> { + if !records(state)? { + return Ok(vec!["bash".into(), "-c".into(), command.into()]); + } + let id = process_id(state)?; + Ok(vec![ + "bash".into(), + "-c".into(), + FOREGROUND_WRAPPER.into(), + "layerfs-exec".into(), + id, + command.into(), + ]) +} + +/// Reads the foreground records in the Sandbox's `/tmp` once. +fn scan_foreground(state: &SandboxState) -> Result { + let (code, out) = runtime::capture( + state, + "/tmp", + vec!["bash".into(), "-c".into(), FOREGROUND_SCAN.into()], + )?; + if code != 0 { + return Err(refused( + "Failed", + format!("foreground record scan exited {code}"), + )); + } + Ok(out) +} + +/// One scanned record line: ` `, with an id this +/// binding generates. A malformed line is ignored. +fn parse_foreground(line: &str) -> Option { + let mut fields = line.split(' '); + let id = fields.next()?; + let path = fields.next()?; + let pid = fields.next()?.parse::().ok()?; + let start = fields.next()?; + if fields.next().is_some() || pid == 0 || path.is_empty() || start.is_empty() { + return None; + } + let (sandbox, token) = id.strip_prefix("proc_")?.split_once('_')?; + if sandbox.is_empty() + || !sandbox.bytes().all(|byte| byte.is_ascii_digit()) + || token.len() != PROCESS_TOKEN_HEX + || !token + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return None; + } + Some(Foreground { + id: id.to_owned(), + path: path.to_owned(), + pid, + start: start.to_owned(), + }) +} + +/// Scanned records of Sandbox `sandbox`, and only those in `path` when it is +/// named. +fn records_in(output: &str, sandbox: &str, path: Option<&str>) -> Vec { + output + .lines() + .filter_map(parse_foreground) + .filter(|record| { + record + .id + .strip_prefix("proc_") + .and_then(|rest| rest.split_once('_')) + .is_some_and(|(owner, _)| owner == sandbox) + }) + .filter(|record| path.is_none_or(|path| record.path == path)) + .collect() +} + +/// Foreground execs still running in the Workspace at `path` of `state`: +/// their record names this Sandbox and Workspace, and `/proc` still shows +/// the pid and start time it recorded. +pub(super) fn foreground_in( + state: &SandboxState, + path: &str, +) -> Result, SdkRefusal> { + let output = scan_foreground(state)?; + Ok(records_in(&output, &state.id, Some(path))) +} diff --git a/api/crates/layerfs-control/src/sandbox/exec/runtime.rs b/api/crates/layerfs-control/src/sandbox/exec/runtime.rs new file mode 100644 index 0000000000..58149fe8c6 --- /dev/null +++ b/api/crates/layerfs-control/src/sandbox/exec/runtime.rs @@ -0,0 +1,50 @@ +//! This Sandbox's runtime behind the two calls the exec modules make: `run` +//! for a command whose output is the result, `capture` for a short probe +//! whose whole stdout is its answer. Both run with the Sandbox's command +//! identity and the scratch directory as home, since that identity has no +//! account in a container. +use crate::sandbox::output::Spool; +use crate::sandbox::registry::SandboxState; +use layerfs_bridge::control::SdkRefusal; + +/// One command to its end: exit code, stdout, stderr. `spool` sends every +/// non-empty stream to its file instead of keeping it inline. +pub(super) fn run( + state: &SandboxState, + directory: &str, + arguments: Vec, + spool: bool, +) -> Result<(i32, Spool, Spool), SdkRefusal> { + let runtime = state.runtime()?; + crate::sandbox::runtime::run( + runtime, + state.identity, + arguments, + vec![format!( + "HOME={}", + crate::sandbox::runtime::scratch(runtime) + )], + directory, + spool, + ) +} + +/// [`run`] of a short probe whose whole stdout is its answer. +pub(super) fn capture( + state: &SandboxState, + directory: &str, + arguments: Vec, +) -> Result<(i32, String), SdkRefusal> { + let runtime = state.runtime()?; + let (code, stdout, _) = crate::sandbox::runtime::capture( + runtime, + state.identity, + arguments, + vec![format!( + "HOME={}", + crate::sandbox::runtime::scratch(runtime) + )], + directory, + )?; + Ok((code, String::from_utf8_lossy(&stdout).into_owned())) +} diff --git a/api/crates/layerfs-control/src/sandbox/route.rs b/api/crates/layerfs-control/src/sandbox/route.rs index 6fea3aceb9..cd04e8a03a 100644 --- a/api/crates/layerfs-control/src/sandbox/route.rs +++ b/api/crates/layerfs-control/src/sandbox/route.rs @@ -348,6 +348,7 @@ pub(crate) fn in_worker(root: &Path, args: &SdkValue, events: &mut dyn FnMut(Sdk | "workspace.exec" | "workspace.commit" | "workspace.unmount" + | "workspace.holders" | "backend.daemon.workspace.status" ) { return Err(refused( diff --git a/api/crates/layerfs-control/src/sandbox/runtime.rs b/api/crates/layerfs-control/src/sandbox/runtime.rs index 34db1c6cd1..0d2e3388b7 100644 --- a/api/crates/layerfs-control/src/sandbox/runtime.rs +++ b/api/crates/layerfs-control/src/sandbox/runtime.rs @@ -239,6 +239,30 @@ pub(crate) fn scratch(runtime: &Sandbox) -> String { runtime.scratch_directory() } +/// One command to its end, its two streams kept whole: used by the short +/// probes and scans that read a command's answer (a foreground record scan, a +/// process-group scope probe), never by an exec whose output is the result. +pub(crate) fn capture( + runtime: &Sandbox, + identity: CommandIdentity, + arguments: Vec, + environment: Vec, + directory: &str, +) -> Result<(i32, Vec, Vec), SdkRefusal> { + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let request = layerfs_sandbox::ExecRequest { + identity, + arguments, + environment, + directory: directory.to_owned(), + stdin: false, + }; + let exit = runtime + .run(&request, &mut stdout, &mut stderr) + .map_err(failed)?; + Ok((exit.code, stdout, stderr)) +} + /// One command to its end: exit code, stdout, stderr. `spool` sends every /// non-empty stream to its file instead of keeping it inline. pub(crate) fn run( diff --git a/api/crates/layerfs-control/src/workspace/calls.rs b/api/crates/layerfs-control/src/workspace/calls.rs index 251141d1ee..702e826758 100644 --- a/api/crates/layerfs-control/src/workspace/calls.rs +++ b/api/crates/layerfs-control/src/workspace/calls.rs @@ -9,7 +9,7 @@ use crate::machine::worker::Unreached; use crate::sandbox::direct; use crate::sandbox::family::sandbox_of; use crate::sandbox::route::{on_machine, reach}; -use layerfs_bridge::control::SdkValue; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; /// Runs one `workspace` call. pub fn workspace(context: &mut Context<'_>, call: &str, args: &SdkValue) -> Outcome { @@ -67,6 +67,33 @@ pub fn workspace(context: &mut Context<'_>, call: &str, args: &SdkValue) -> Outc ("directory", SdkValue::Text(directory)), ]; } + // A workspace-scoped kill, and the force gate, run where the Sandbox's + // runtime is: its machine's worker or this process, even when this host + // reaches only its daemon's forwarded port (#535 G4). + let ending = match call { + "unmount" => Some(crate::workspace::serve::ending(args)?), + _ => None, + }; + let killing = ending == Some(crate::workspace::serve::Ending::Kill); + if killing || ending == Some(crate::workspace::serve::Ending::Normal { force: true }) { + let at = ws.at.as_str(); + // a name is resolved here where the Workspace records are; the + // machine side resolves one it recorded itself + let path = match forwarded && !at.starts_with('/') { + true => direct::exec_directory(&home, &record, at)?, + false => at.to_owned(), + }; + match holders(context, &record, &path, killing) { + Ok(Some((path, ids))) => { + return Err(crate::sandbox::exec::busy(record.name(), &path, ids)) + } + Ok(None) => {} + // a scan that cannot run leaves the recovery path (`force`) as it + // was, exactly as before this option existed + Err(_) if !killing => {} + Err(refusal) => return Err(refusal), + } + } let op = match call { "exec" => { more.push(("command", args.get("command").cloned().unwrap_or_default())); @@ -82,7 +109,7 @@ pub fn workspace(context: &mut Context<'_>, call: &str, args: &SdkValue) -> Outc "workspace.commit" } "unmount" => { - for flag in ["discard", "force"] { + for flag in ["discard", "force", "kill"] { if let Some(value) = args.get(flag) { more.push((flag, value.clone())); } @@ -99,7 +126,30 @@ pub fn workspace(context: &mut Context<'_>, call: &str, args: &SdkValue) -> Outc reach(context, &record, op, more, false).unwrap_or_else(|Unreached(why)| Err(why)) } _ => on_machine(context, &record, op, more), - }?; + }; + // A normal unmount the daemon refused Busy names the executor's foreground + // execs, so the caller can cancel them instead of forcing. + let out = match out { + Err(refusal) + if call == "unmount" + && refusal.code == "Busy" + && refusal.detail.get("processes").is_none() => + { + let at = ws.at.as_str(); + let path = match at.starts_with('/') { + true => at.to_owned(), + false => { + direct::exec_directory(&home, &record, at).unwrap_or_else(|_| at.to_owned()) + } + }; + match holders(context, &record, &path, false) { + Ok(Some((path, ids))) => Err(crate::sandbox::exec::busy(record.name(), &path, ids)), + _ => Err(refusal), + } + } + other => other, + }; + let out = out?; if call == "exec" { // the command's output as events too, for front ends that stream for stream in ["stdout", "stderr"] { @@ -114,3 +164,30 @@ pub fn workspace(context: &mut Context<'_>, call: &str, args: &SdkValue) -> Outc } Ok(out) } + +/// The foreground execs still running in the Workspace at `path`, from the +/// side that runs the Sandbox's runtime: `{ processes }` read, or cancelled +/// first (`kill`). `Ok(None)` when none is left. +fn holders( + context: &mut Context<'_>, + record: &crate::sandbox::record::SandboxRecord, + path: &str, + kill: bool, +) -> Result)>, SdkRefusal> { + let more = vec![ + ("path", SdkValue::text(path)), + ("kill", SdkValue::Bool(kill)), + ]; + let answer = on_machine(context, record, "workspace.holders", more)?; + let ids = match answer.get("processes") { + Some(SdkValue::List(items)) => (items.iter()) + .filter_map(|item| match item { + SdkValue::Text(id) => Some(id.clone()), + _ => None, + }) + .collect::>(), + _ => Vec::new(), + }; + let path = answer.get_text("path").unwrap_or(path).to_owned(); + Ok((!ids.is_empty()).then_some((path, ids))) +} diff --git a/api/crates/layerfs-control/src/workspace/serve.rs b/api/crates/layerfs-control/src/workspace/serve.rs index b5b257eda7..f3014fff95 100644 --- a/api/crates/layerfs-control/src/workspace/serve.rs +++ b/api/crates/layerfs-control/src/workspace/serve.rs @@ -239,8 +239,44 @@ pub(crate) fn commit(args: &SdkValue) -> Outcome { values::commit(&reply) } -/// `workspace.unmount(ws, { discard?, force? })`: the daemon's answer. +/// How one `workspace.unmount` call ends the Workspace. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Ending { + /// The daemon's normal unmount; `force` aborts a live connection instead. + Normal { force: bool }, + /// Cancel only the holders of this Workspace, then a normal unmount. + Kill, +} + +/// Reads the `force` and `kill` arguments: `kill` accepts only `"workspace"`, +/// it cannot be combined with `force`, and a missing `kill` leaves the +/// existing force path alone. The caller's side (`workspace::calls`) uses it +/// to cancel holders where the runtime is; this side uses it to refuse an +/// invalid combination before any effect. +pub(crate) fn ending(args: &SdkValue) -> Result { + let force = args.get_bool("force"); + match args.get_text("kill") { + None => Ok(Ending::Normal { force }), + Some("workspace") if !force => Ok(Ending::Kill), + Some("workspace") => Err(crate::common::outcome::refused( + "InvalidInput", + "force and kill cannot be combined: kill: \"workspace\" cancels only this Workspace's holders, then unmounts it normally", + )), + Some(other) => Err(crate::common::outcome::refused( + "InvalidInput", + format!( + "kill must be \"workspace\", not {other:?}: only a Workspace's own holders are killed, never another Workspace or the Sandbox" + ), + )), + } +} + +/// `workspace.unmount(ws, { discard?, force?, kill? })`: the daemon's answer. +/// `kill: "workspace"` first cancels only this Workspace's foreground execs, +/// then unmounts normally; it never stops or deletes the Sandbox and never +/// touches another Workspace. pub(crate) fn unmount(args: &SdkValue) -> Outcome { + let ending = ending(args)?; let state = sandbox(args)?; let path = text(args, "path")?; if let Some(value) = crate::workspace::unmount_guard::find(&state, path)? { @@ -266,7 +302,7 @@ pub(crate) fn unmount(args: &SdkValue) -> Outcome { let mut api = workspace(&mut control); // Forced: one abort write ends the connection, also after a normal // unmount stalled at Join; uncommitted changes are dropped. - if args.get_bool("force") { + if ending == (Ending::Normal { force: true }) { match api.unmount(token, true).map_err(operation)? { Some(outcome) => Ok(values::force_unmounted(&outcome)), None => values::unmounted(&Unmounted::Closed), diff --git a/core/docs/issues/538/workspace-kill-architecture.png b/core/docs/issues/538/workspace-kill-architecture.png new file mode 100644 index 0000000000..e2cb5e0ab3 Binary files /dev/null and b/core/docs/issues/538/workspace-kill-architecture.png differ diff --git a/core/docs/issues/538/workspace-kill-sequence.png b/core/docs/issues/538/workspace-kill-sequence.png new file mode 100644 index 0000000000..98ca14b5f4 Binary files /dev/null and b/core/docs/issues/538/workspace-kill-sequence.png differ diff --git a/engine/crates/layerfs-persistence/tests/sqlite_publication.rs b/engine/crates/layerfs-persistence/tests/sqlite_publication.rs index 8d0f4fa772..3806e3c545 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_publication.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_publication.rs @@ -56,15 +56,12 @@ fn unit(pack_id: i64, objects: usize) -> Publication { } } #[test] -fn wal_full_settings_and_unavailable_engine_are_explicit() { +fn wal_full_settings_are_explicit() { let t = Temp::new("durable-profile"); let path = t.join("db"); - let c = PersistenceConfig::sqlite(&path); - assert_eq!( - SqliteHandles::create(c, StoragePolicy::frozen_default(), &config()).err(), - Some(PersistenceError::BackendUnavailable) - ); - assert!(!path.exists()); + // A SQLite handle always has an engine on macOS and Linux: the old + // `BackendUnavailable` case selected a fake PostgreSQL backend, which + // `SqliteHandles` no longer has (#535). let h = create(&path); let p = h.profile(); assert_eq!(p.identity, "sqlite-wal-full-v2");