fix: complete gate to green -- fork ledger flush (#243), coverage, Linux Rust base, release gh login

CHANGELOG.md

@@ -191,6 +191,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- The global session index (`main.db`) now always lives in the Capsem home's
+  `sessions/` directory. The service derived it from the run directory's
+  parent, so a service started with `CAPSEM_RUN_DIR` outside the home put it
+  somewhere else -- every such service under `/tmp` shared
+  `/tmp/sessions/main.db`, and a copy left in an older ledger format stopped
+  each new service at startup. The default layout is unchanged.
+
+- Forking a running sandbox now carries everything its session ledger
+  recorded up to the fork. The ledger writer keeps accepted rows in memory
+  until its next disk flush, up to five seconds away, and the fork copied the
+  file without asking it to flush, so a fork left out the source's last few
+  seconds of network, tool, exec and security events. The fork now flushes
+  the writer while the guest is frozen, and fails rather than producing an
+  incomplete ledger if that flush fails (google/capsem#243).
+
 - The guest rootfs OBOM (`obom.cdx.json`) now describes the rootfs exactly
   and builds about ten times faster. The build unpacked the rootfs on the
   build host before scanning it: on macOS the case-insensitive filesystem

build_system/builder/gate/pidfiles.py

@@ -123,7 +123,7 @@ def stop(pidfile: Path, settings: gate_config.PidfileConfig) -> None:
     but a *named* process that survives both signals is.
     """
     pidfile = Path(pidfile)
-    recorded = _recorded_pid(pidfile)
+    recorded = recorded_pid(pidfile)
 
     if recorded is not None and running(recorded, settings):
         os.kill(recorded, signal.SIGTERM)
@@ -137,7 +137,7 @@ def stop(pidfile: Path, settings: gate_config.PidfileConfig) -> None:
     pidfile.unlink(missing_ok=True)
 
 
-def _recorded_pid(pidfile: Path) -> int | None:
+def recorded_pid(pidfile: Path) -> int | None:
     try:
         recorded = pidfile.read_text(encoding="utf-8").strip()
     except (OSError, ValueError):

build_system/builder/gate/service.py

@@ -151,6 +151,7 @@ def perform(self, context: Context) -> None:
         directory = self._directory or run_dir(context.config)
         path = directory / settings.socket
 
+        pidfile = directory / settings.pidfile
         for _ in range(settings.ready_attempts):
             if path.exists():
                 probe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
@@ -161,6 +162,14 @@ def perform(self, context: Context) -> None:
                     pass
                 finally:
                     probe.close()
+            # A daemon that died is not a slow one: say so now, rather than
+            # spend the budget and report a connection it could never accept.
+            pid = pidfiles.recorded_pid(pidfile)
+            if pid is not None and not pidfiles.running(pid, context.config.pidfiles):
+                raise GateError(
+                    f"capsem-service (pid {pid}) exited before it listened on {path}; "
+                    f"its log is in {directory}"
+                )
             time.sleep(settings.ready_interval_seconds)
 
         raise GateError(

build_system/builder/release/tools/fetch_channel_source_manifest.py

@@ -136,6 +136,26 @@ def resolve_source_manifest(
     return payload, source
 
 
+def github_token() -> str | None:
+    """`GITHUB_TOKEN` if exported, otherwise the operator's `gh` login.
+
+    The release dispatches its hosted lane through `gh`, so that login is
+    already required; demanding a second export found its absence only after
+    the release's local suites had spent eight minutes passing.
+    """
+    exported = os.environ.get("GITHUB_TOKEN", "")
+    if exported:
+        return exported
+    try:
+        result = subprocess.run(
+            ["gh", "auth", "token"], capture_output=True, text=True, check=False, timeout=30
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    token = result.stdout.strip() if result.returncode == 0 else ""
+    return token or None
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--channel", required=True)
@@ -168,9 +188,13 @@ def main() -> int:
     fallback_url = args.fallback_url or (
         f"https://release.capsem.org/assets/{args.channel}/manifest.json"
     )
-    token = os.environ.get("GITHUB_TOKEN", "")
+    token = github_token()
     if not token:
-        print("GITHUB_TOKEN is required to resolve source manifests", file=sys.stderr)
+        print(
+            "resolving source manifests needs GitHub credentials: export GITHUB_TOKEN "
+            "or log in with `gh auth login`",
+            file=sys.stderr,
+        )
         return 1
     try:
         retired_graph: retirement.RetiredPublicGraph | None = None

build_system/docker/Dockerfile.linux-rust-base

@@ -19,6 +19,9 @@ FROM ${BASE}
 # exists for.
 COPY Cargo.toml Cargo.lock rust-toolchain.toml /src/
 COPY crates /src/crates
+# Every workspace member, or `cargo fetch --locked` cannot load the workspace;
+# tests/citadel/test_linux_rust_base_copies_the_workspace.py holds the list.
+COPY sdk/rust /src/sdk/rust
 
 WORKDIR /src
 
@@ -61,6 +64,20 @@ RUN cd /src/web/app \
     && pnpm run build \
     && test -s /src/web/app/dist/index.html
 
+# The build-system Python environment, for the Rust tests that hand an export
+# to a reference reader (`warcio`) through `uv run --project build_system`.
+# Built here, with network, into /src/build_system/.venv; the lane's source
+# copy excludes `.venv`, so it survives, and `uv run --frozen` in the sealed
+# lane finds it complete. Without it those tests tried to download packages
+# inside a lane that has none and failed on DNS.
+#
+# The uv cache lives under /src too, handed to the lane user below: the lane
+# copies a newer build_system/pyproject.toml over this one, and a rebuild of the editable
+# project needs its build backend (setuptools) from a cache it can read.
+ENV UV_CACHE_DIR=/src/.uv-cache
+COPY build_system /src/build_system
+RUN cd /src && uv sync --project build_system --frozen
+
 # Ownership last, so it covers everything every step above created as root.
 # A non-root user is required: the suite chmods an asset to 0o000 and demands
 # the read fail, and root ignores permissions. `ubuntu:24.04` already ships a

build_system/tests/gate/test_gate_smoke_lifecycle.py

@@ -170,3 +170,27 @@ def test_a_failure_preserves_evidence_before_anything_is_released(
         raise GateError("boom")
 
     assert order.index("preserve") < order.index("release")
+
+
+def test_a_service_that_exits_before_listening_is_reported_at_once(tmp_path: Path) -> None:
+    """A daemon that died is not a slow daemon.
+
+    capsem-service refused a stale main.db and exited within a second, and the
+    wait spent the whole readiness budget before calling it "did not accept a
+    connection" -- which sent the diagnosis looking for a hang.
+    """
+    import subprocess
+    import time
+
+    from capsem_builder.gate.context import Context
+    from capsem_builder.gate.service import WaitForSocket
+
+    exited = subprocess.Popen(["true"])
+    exited.wait()
+    (tmp_path / CONFIG.service.pidfile).write_text(str(exited.pid), encoding="utf-8")
+
+    started = time.monotonic()
+    with pytest.raises(GateError, match="exited before it listened"):
+        WaitForSocket(tmp_path).perform(Context(RecordingRunner(PROJECT_ROOT), CONFIG))
+    budget = CONFIG.service.ready_attempts * CONFIG.service.ready_interval_seconds
+    assert time.monotonic() - started < budget / 2, "it waited out the readiness budget"

build_system/tests/helpers/integration_test.py

@@ -86,7 +86,8 @@ def _integration_runtime_root() -> Path:
 
 CAPSEM_HOME = INTEGRATION_HOME
 PERSISTENT_DIR = INTEGRATION_RUN_DIR / "persistent"
-MAIN_DB = INTEGRATION_RUNTIME_ROOT / "sessions" / "main.db"
+# The service keeps main.db in its home, never beside the run directory.
+MAIN_DB = CAPSEM_HOME / "sessions" / "main.db"
 SERVICE_SOCKET = INTEGRATION_RUN_DIR / "service.sock"
 SERVICE_PIDFILE = INTEGRATION_RUN_DIR / "service.pid"
 

build_system/tests/release/test_channel_source_credentials.py

@@ -0,0 +1,54 @@
+"""Resolving a channel's source manifest uses the release's own GitHub login.
+
+`release-binaries` dispatches its hosted lane through `gh`, so the operator's
+`gh` login is already a release prerequisite. The source-manifest fetch alone
+also demanded `GITHUB_TOKEN` in the environment, and found out it was missing
+eight minutes in, after the release's citadel, contract and build-system
+suites had all passed.
+"""
+
+from __future__ import annotations
+
+import subprocess
+
+import pytest
+from capsem_builder.release.tools import fetch_channel_source_manifest as fetch
+
+
+def test_an_exported_token_is_used_as_is(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("GITHUB_TOKEN", "exported")
+    monkeypatch.setattr(fetch.subprocess, "run", pytest.fail)
+    assert fetch.github_token() == "exported"
+
+
+def test_without_one_the_gh_login_supplies_it(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("GITHUB_TOKEN", raising=False)
+    calls: list[list[str]] = []
+
+    def run(command, **kwargs):
+        calls.append(command)
+        return subprocess.CompletedProcess(command, 0, stdout="from-gh\n", stderr="")
+
+    monkeypatch.setattr(fetch.subprocess, "run", run)
+    assert fetch.github_token() == "from-gh"
+    assert calls == [["gh", "auth", "token"]]
+
+
+@pytest.mark.parametrize(
+    "outcome",
+    [
+        subprocess.CompletedProcess(["gh"], 1, stdout="", stderr="not logged in"),
+        subprocess.CompletedProcess(["gh"], 0, stdout="\n", stderr=""),
+        FileNotFoundError("gh"),
+    ],
+)
+def test_no_token_anywhere_is_none(monkeypatch: pytest.MonkeyPatch, outcome) -> None:
+    monkeypatch.delenv("GITHUB_TOKEN", raising=False)
+
+    def run(command, **kwargs):
+        if isinstance(outcome, BaseException):
+            raise outcome
+        return outcome
+
+    monkeypatch.setattr(fetch.subprocess, "run", run)
+    assert fetch.github_token() is None

build_system/tests/scripts/test_integration_script_profiles.py

@@ -182,7 +182,7 @@ def test_integration_script_service_paths_use_process_scoped_isolated_home():
     assert module.INTEGRATION_RUN_DIR == module.INTEGRATION_RUNTIME_ROOT / "run"
     assert module.SERVICE_SOCKET == module.INTEGRATION_RUN_DIR / "service.sock"
     assert module.PERSISTENT_DIR == module.INTEGRATION_RUN_DIR / "persistent"
-    assert module.MAIN_DB == module.INTEGRATION_RUNTIME_ROOT / "sessions" / "main.db"
+    assert module.MAIN_DB == module.CAPSEM_HOME / "sessions" / "main.db"
     assert len(os.fsencode(module.SERVICE_SOCKET)) < 108
 
 

build_system/tests/test_test_ownership.py

@@ -91,6 +91,7 @@
         "build_system/tests/packaging/test_macos_packaging_boundary.py",
         "build_system/tests/packaging/test_shared_packaging_boundary.py",
         "build_system/tests/policy/test_policy_modules.py",
+        "build_system/tests/release/test_channel_source_credentials.py",
         "build_system/tests/release/test_deb_package_portability.py",
         "build_system/tests/release/test_release_module_boundary.py",
         "build_system/tests/release/test_release_foundation_tool_boundary.py",

config/gate.toml

@@ -1590,7 +1590,7 @@ lane_tag = "capsem-linux-rust:latest"
 # Not the workspace manifests: `cargo fetch --locked` fetches everything the
 # lock resolves regardless of features, so a manifest edit that leaves
 # `Cargo.lock` alone leaves the sealed lane with every source it needs.
-identity_inputs = ["Cargo.lock", "rust-toolchain.toml", "sdk/typescript/pnpm-lock.yaml", "web/app/pnpm-lock.yaml"]
+identity_inputs = ["Cargo.lock", "rust-toolchain.toml", "sdk/typescript/pnpm-lock.yaml", "web/app/pnpm-lock.yaml", "build_system/pyproject.toml", "build_system/uv.lock"]
 # Named here so the lane's refusal and the recipe that answers it cannot drift
 # apart. `test_every_recipe_the_gate_tells_an_operator_to_run_exists` checks
 # this resolves to a real recipe -- the last time a message named one that did
@@ -2368,7 +2368,7 @@ rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent
 # macOS compiles a different cfg-selected line inventory. Its stronger floors
 # must not raise Linux thresholds using code that Linux cannot compile or test.
 # Overrides may only increase the shared floors; no platform gets an exemption.
-rust_coverage_platform_crate_floors = { Linux = {}, Darwin = { capsem-agent = 50.29, capsem-assets = 90.3, capsem-bench = 50.56, capsem-core = 81.0, capsem-foundation = 81.0, capsem-mock-server = 58.6, capsem-network = 96.62, capsem-process = 45.18, capsem-proto = 91.54, capsem-router = 73.73, capsem-service = 69.51 } }
+rust_coverage_platform_crate_floors = { Linux = {}, Darwin = { capsem = 51.27, capsem-agent = 53.13, capsem-api = 95.34, capsem-assets = 90.3, capsem-bench = 50.56, capsem-config = 89.69, capsem-core = 81.0, capsem-foundation = 83.8, capsem-logger = 91.88, capsem-mcp-builtin = 56.56, capsem-mock-server = 58.6, capsem-network = 97.5, capsem-process = 48.12, capsem-proto = 91.54, capsem-router = 79.8, capsem-service = 74.73 } }
 # Nextest reads its profile from the environment; `--profile` on the command
 # line belongs to cargo-llvm-cov and would select a build profile instead.
 rust_test_profile_variable = "NEXTEST_PROFILE"

crates/capsem-core/src/fs_monitor/tests.rs

@@ -714,18 +714,40 @@ fn pin_mtime(path: &Path) {
     assert_eq!(rc, 0, "utimensat: {}", std::io::Error::last_os_error());
 }
 
+fn ctime(path: &Path) -> (i64, i64) {
+    use std::os::unix::fs::MetadataExt;
+    let metadata = std::fs::metadata(path).unwrap();
+    (metadata.ctime(), metadata.ctime_nsec())
+}
+
 /// The forgery this guards: rewrite a file in place to the same length, then
 /// put its mtime back. Size and mtime say nothing happened; ctime does.
+///
+/// ctime is only as fine as the kernel's timestamp clock. Linux stamps inodes
+/// from a coarse clock that advances every few milliseconds, so two writes in
+/// one tick share a ctime and no metadata can tell them apart. The rewrite is
+/// repeated until the kernel records a new ctime -- the precondition this test
+/// is about -- rather than assuming a later write always gets one.
 #[test]
 fn an_in_place_rewrite_with_a_restored_mtime_is_still_a_modification() {
     let root = tempfile::tempdir().unwrap();
     let path = root.path().join("payload.bin");
     std::fs::write(&path, b"aaaaa").unwrap();
     pin_mtime(&path);
     let before = workspace_snapshot(&ContainedDir::open_root(root.path()).unwrap());
+    let first = ctime(&path);
 
-    std::fs::write(&path, b"bbbbb").unwrap();
-    pin_mtime(&path);
+    let mut attempts = 0;
+    loop {
+        std::fs::write(&path, b"bbbbb").unwrap();
+        pin_mtime(&path);
+        if ctime(&path) != first {
+            break;
+        }
+        attempts += 1;
+        assert!(attempts < 1_000, "the kernel never advanced ctime");
+        std::thread::sleep(std::time::Duration::from_millis(1));
+    }
     let after = workspace_snapshot(&ContainedDir::open_root(root.path()).unwrap());
 
     assert_eq!(

crates/capsem-mcp-builtin/src/main.rs

@@ -237,6 +237,37 @@ fn extract_text(resp: JsonRpcResponse) -> Result<String, String> {
     }
 }
 
+// -- Startup --
+
+/// Which member of a pooled server this process is; peer 0 is the singleton.
+fn peer_index(raw: Option<String>) -> u32 {
+    raw.and_then(|value| value.parse().ok()).unwrap_or(0)
+}
+
+/// The per-peer singleton lock, so pool members do not fight over one file.
+fn lock_file_name(peer_index: u32) -> String {
+    if peer_index == 0 {
+        "mcp-builtin.lock".to_string()
+    } else {
+        format!("mcp-builtin-{peer_index}.lock")
+    }
+}
+
+/// Read, parse and validate the active profile, naming the file in any error.
+fn load_active_profile(path: &str) -> Result<ActiveProfileFile> {
+    let text = std::fs::read_to_string(path)
+        .map_err(anyhow::Error::new)
+        .with_context(|| format!("read active profile {path}"))?;
+    let profile: ActiveProfileFile = toml::from_str(&text)
+        .map_err(anyhow::Error::new)
+        .with_context(|| format!("parse active profile {path}"))?;
+    profile
+        .validate()
+        .map_err(anyhow::Error::msg)
+        .with_context(|| format!("validate active profile {path}"))?;
+    Ok(profile)
+}
+
 // -- Main --
 
 #[tokio::main]
@@ -262,18 +293,10 @@ async fn main() -> Result<()> {
     // Per-peer index for pool members (set by the aggregator's
     // connect_stdio when spawning peer 1..N of a pooled server). Each
     // peer gets its own lockfile so they don't fight over the singleton.
-    let peer_index: u32 = std::env::var("CAPSEM_BUILTIN_PEER_INDEX")
-        .ok()
-        .and_then(|s| s.parse().ok())
-        .unwrap_or(0);
+    let peer_index = peer_index(std::env::var("CAPSEM_BUILTIN_PEER_INDEX").ok());
 
     if let (Some(pid), Some(dir)) = (parent_pid, session_dir) {
-        let lock_name = if peer_index == 0 {
-            "mcp-builtin.lock".to_string()
-        } else {
-            format!("mcp-builtin-{peer_index}.lock")
-        };
-        let lock_path = std::path::PathBuf::from(dir).join(&lock_name);
+        let lock_path = std::path::PathBuf::from(dir).join(lock_file_name(peer_index));
         match capsem_guard::install(Some(pid), &lock_path) {
             Ok(Some(guards)) => {
                 // Keep the guards alive for the process's lifetime.
@@ -292,16 +315,7 @@ async fn main() -> Result<()> {
 
     let active_profile_path =
         std::env::var("CAPSEM_ACTIVE_PROFILE").map_err(|_| anyhow::anyhow!("CAPSEM_ACTIVE_PROFILE is required"))?;
-    let active_profile_text = std::fs::read_to_string(&active_profile_path)
-        .map_err(anyhow::Error::new)
-        .with_context(|| format!("read active profile {active_profile_path}"))?;
-    let active_profile: ActiveProfileFile = toml::from_str(&active_profile_text)
-        .map_err(anyhow::Error::new)
-        .with_context(|| format!("parse active profile {active_profile_path}"))?;
-    active_profile
-        .validate()
-        .map_err(anyhow::Error::msg)
-        .with_context(|| format!("validate active profile {active_profile_path}"))?;
+    let active_profile = load_active_profile(&active_profile_path)?;
     let security_rules = Arc::new(active_profile.compile_security_rule_set().map_err(anyhow::Error::msg)?);
     let plugin_policy = Arc::new(active_profile.plugins.clone());