diff --git a/MANIFEST b/MANIFEST index 27d0251ab..0ee89fe22 100644 --- a/MANIFEST +++ b/MANIFEST @@ -51,9 +51,8 @@ /etc/os-release f 0644 0 0 /etc/profile f 0644 0 0 /etc/motd f 0644 0 0 -# services matches the ad-hoc table in -# apps/browser-demos/lib/browser-kernel.ts byte-for-byte so the cutover -# is invisible to getservbyname/getservbyport callers. +# The rootfs file is the authoritative services database. Rootfs-derived +# images inherit it rather than rebuilding or synthesizing a second table. /etc/services f 0644 0 0 # ── Shells ───────────────────────────────────────────────────────── diff --git a/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts b/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts index b5f020029..49905397d 100644 --- a/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts +++ b/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts @@ -1150,6 +1150,15 @@ async function bootProfile( maxMemoryPages: profile.init?.maxMemoryPages, onStdout: (data) => recordProcessOutput(data, "stdout"), onStderr: (data) => recordProcessOutput(data, "stderr"), + onHostDiagnostic: (diagnostic) => { + if (!isCurrent()) return; + host.pushDmesg({ + t: bootElapsedMs(bootStartedAt), + level: "warn", + facility: "kernel", + msg: diagnostic.message, + }); + }, onProcessEvent: (event) => { if (isCurrent()) host.emitProcessEvent(event); }, onHttpBridgePendingRequests: (count) => { if (isCurrent()) host.setWebPreviewPendingRequests(count); diff --git a/crates/fork-instrument/src/main.rs b/crates/fork-instrument/src/main.rs index dde9b50a9..8f63f5498 100644 --- a/crates/fork-instrument/src/main.rs +++ b/crates/fork-instrument/src/main.rs @@ -13,7 +13,9 @@ use anyhow::{Context, Result}; use clap::Parser; use std::fs; -use std::path::PathBuf; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; use fork_instrument::{Options, analyze, instrument}; @@ -65,16 +67,48 @@ fn main() -> Result<()> { let output_path = cli.output.as_ref().ok_or_else(|| { anyhow::anyhow!("--output is required unless --discover-only is set") })?; + // Capture this before writing: `--output` is allowed to name the input + // file, and output creation/truncation must not become the source of truth + // for the executable mode we are preserving. + let input_mode = input_mode(&cli.input)?; let output = instrument(&input, &opts) .with_context(|| format!("instrumenting {}", cli.input.display()))?; fs::write(output_path, &output) .with_context(|| format!("writing output: {}", output_path.display()))?; + preserve_input_mode(input_mode, output_path)?; Ok(()) } +#[cfg(unix)] +fn input_mode(input_path: &Path) -> Result { + let mode = fs::metadata(input_path) + .with_context(|| format!("stat input for permissions: {}", input_path.display()))? + .permissions() + .mode(); + Ok(mode) +} + +#[cfg(not(unix))] +fn input_mode(_input_path: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn preserve_input_mode(input_mode: u32, output_path: &Path) -> Result<()> { + let permissions = fs::Permissions::from_mode(input_mode); + fs::set_permissions(output_path, permissions) + .with_context(|| format!("setting output permissions: {}", output_path.display()))?; + Ok(()) +} + +#[cfg(not(unix))] +fn preserve_input_mode(_input_mode: (), _output_path: &Path) -> Result<()> { + Ok(()) +} + fn print_analysis_json(analysis: &fork_instrument::Analysis) { // Hand-rolled JSON to avoid a serde dependency for a tiny output. // Format is one-entry-per-line array of `{name, is_import}` objects. diff --git a/crates/fork-instrument/tests/cli_permissions.rs b/crates/fork-instrument/tests/cli_permissions.rs new file mode 100644 index 000000000..5097e5d6c --- /dev/null +++ b/crates/fork-instrument/tests/cli_permissions.rs @@ -0,0 +1,95 @@ +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(0); + +const INPUT_WAT: &str = r#" +(module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (memory 1) + (func (export "_start") + (drop (call $fork)))) +"#; + +struct TempDir(PathBuf); + +impl TempDir { + fn new(test_name: &str) -> Self { + let id = NEXT_TEMP_ID.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "kandelo-fork-instrument-{test_name}-{}-{id}", + std::process::id() + )); + fs::create_dir(&path).expect("create test directory"); + Self(path) + } + + fn join(&self, path: impl AsRef) -> PathBuf { + self.0.join(path) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn write_input(path: &Path, mode: u32) { + fs::write(path, wat::parse_str(INPUT_WAT).expect("compile input WAT")) + .expect("write input Wasm"); + fs::set_permissions(path, fs::Permissions::from_mode(mode)).expect("set input mode"); +} + +fn run_instrumenter(input: &Path, output: &Path) { + let status = Command::new(env!("CARGO_BIN_EXE_wasm-fork-instrument")) + .arg(input) + .arg("--output") + .arg(output) + .status() + .expect("run wasm-fork-instrument"); + assert!(status.success(), "instrumenter exited with {status}"); +} + +fn mode(path: &Path) -> u32 { + fs::metadata(path) + .expect("stat output") + .permissions() + .mode() + & 0o7777 +} + +#[test] +fn new_output_preserves_input_mode() { + let dir = TempDir::new("new-output"); + let input = dir.join("input.wasm"); + let output = dir.join("output.wasm"); + write_input(&input, 0o751); + + run_instrumenter(&input, &output); + + assert_eq!(mode(&input), 0o751); + assert_eq!(mode(&output), 0o751); + wasmparser::Validator::new() + .validate_all(&fs::read(output).expect("read output")) + .expect("instrumented output validates"); +} + +#[test] +fn in_place_output_preserves_input_mode() { + let dir = TempDir::new("in-place"); + let input = dir.join("program.wasm"); + write_input(&input, 0o711); + + run_instrumenter(&input, &input); + + assert_eq!(mode(&input), 0o711); + wasmparser::Validator::new() + .validate_all(&fs::read(input).expect("read output")) + .expect("instrumented output validates"); +} diff --git a/crates/kernel/src/syscalls.rs b/crates/kernel/src/syscalls.rs index b1c9d67fc..0c9a47511 100644 --- a/crates/kernel/src/syscalls.rs +++ b/crates/kernel/src/syscalls.rs @@ -5546,13 +5546,42 @@ pub fn sys_exit(proc: &mut Process, host: &mut dyn HostIO, status: i32) { proc.exit_signal = 0; } -/// Get the current time from the specified clock. +fn is_encoded_process_cpu_clock_id(clock_id: u32) -> bool { + // musl/Linux encode process CPU clocks as (-pid-1)*8 + 2. Real encoded + // IDs are negative clockid_t values; checking only the low three bits + // would incorrectly accept positive IDs such as 10 (produced when an + // invalid negative pid is passed to clock_getcpuclockid()). + (clock_id as i32) < 0 && (clock_id & 7) == 2 +} + +fn host_clock_id(clock_id: u32) -> Result { + use wasm_posix_shared::clock::*; + + match clock_id { + CLOCK_REALTIME | CLOCK_MONOTONIC | CLOCK_PROCESS_CPUTIME_ID + | CLOCK_THREAD_CPUTIME_ID | CLOCK_BOOTTIME => Ok(clock_id), + // Linux exposes coarse variants to libc consumers such as MariaDB. + // Kandelo's hosts do not maintain separate coarse clock sources, so + // preserve the clock domain while using the corresponding canonical + // source. Returning EINVAL here leaves MariaDB spinning before its + // first filesystem operation. + CLOCK_REALTIME_COARSE => Ok(CLOCK_REALTIME), + CLOCK_MONOTONIC_COARSE => Ok(CLOCK_MONOTONIC), + // clock_getcpuclockid() encodes a process clock as (-pid-1)*8 + 2. + // Kandelo does not yet account CPU time per process, so preserve the + // documented elapsed-time approximation without letting the host + // mistake an encoded ID for CLOCK_REALTIME. + id if is_encoded_process_cpu_clock_id(id) => Ok(CLOCK_PROCESS_CPUTIME_ID), + _ => Err(Errno::EINVAL), + } +} + pub fn sys_clock_gettime( _proc: &Process, host: &mut dyn HostIO, clock_id: u32, ) -> Result { - let (sec, nsec) = host.host_clock_gettime(clock_id)?; + let (sec, nsec) = host.host_clock_gettime(host_clock_id(clock_id)?)?; Ok(WasmTimespec { tv_sec: sec, tv_nsec: nsec, @@ -5576,13 +5605,17 @@ pub fn sys_nanosleep( pub fn sys_clock_getres(_proc: &Process, clock_id: u32) -> Result { use wasm_posix_shared::clock::*; match clock_id { - CLOCK_REALTIME | CLOCK_MONOTONIC | CLOCK_PROCESS_CPUTIME_ID | CLOCK_THREAD_CPUTIME_ID => { - Ok(WasmTimespec { - tv_sec: 0, - tv_nsec: 1_000_000, - }) // 1ms - } - id if (id & 7) == 2 => { + CLOCK_REALTIME + | CLOCK_MONOTONIC + | CLOCK_PROCESS_CPUTIME_ID + | CLOCK_THREAD_CPUTIME_ID + | CLOCK_REALTIME_COARSE + | CLOCK_MONOTONIC_COARSE + | CLOCK_BOOTTIME => Ok(WasmTimespec { + tv_sec: 0, + tv_nsec: 1_000_000, + }), // 1ms + id if is_encoded_process_cpu_clock_id(id) => { // Per-process CPU clock: clock_getcpuclockid encodes as (-pid-1)*8 + 2 Ok(WasmTimespec { tv_sec: 0, @@ -5593,8 +5626,8 @@ pub fn sys_clock_getres(_proc: &Process, clock_id: u32) -> Result Result<(), Errno> { use wasm_posix_shared::clock::*; // Validate clock_id - if clock_id != CLOCK_REALTIME && clock_id != CLOCK_MONOTONIC { + if clock_id != CLOCK_REALTIME && clock_id != CLOCK_MONOTONIC && clock_id != CLOCK_BOOTTIME { return Err(Errno::EINVAL); } // Validate timespec @@ -15824,6 +15857,65 @@ mod tests { assert_eq!(ts.tv_nsec, 123456789); } + #[test] + fn test_clock_gettime_supports_boottime_and_encoded_process_clock() { + let proc = Process::new(1); + let mut host = MockHostIO::new(); + + assert!(sys_clock_gettime( + &proc, + &mut host, + wasm_posix_shared::clock::CLOCK_BOOTTIME, + ) + .is_ok()); + // Linux's clock_getcpuclockid() encoding for PID 1. + assert!(sys_clock_gettime(&proc, &mut host, (-2_i32 * 8 + 2) as u32).is_ok()); + } + + #[test] + fn test_clock_calls_map_linux_coarse_clocks_to_supported_sources() { + use wasm_posix_shared::clock::{ + CLOCK_MONOTONIC, CLOCK_MONOTONIC_COARSE, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, + }; + + let proc = Process::new(1); + let mut host = MockHostIO::new(); + + assert_eq!(host_clock_id(CLOCK_REALTIME_COARSE), Ok(CLOCK_REALTIME)); + assert_eq!(host_clock_id(CLOCK_MONOTONIC_COARSE), Ok(CLOCK_MONOTONIC),); + for clock_id in [CLOCK_REALTIME_COARSE, CLOCK_MONOTONIC_COARSE] { + assert!(sys_clock_gettime(&proc, &mut host, clock_id).is_ok()); + assert!(sys_clock_getres(&proc, clock_id).is_ok()); + } + } + + #[test] + fn test_clock_gettime_rejects_unknown_clock() { + let proc = Process::new(1); + let mut host = MockHostIO::new(); + assert!(matches!( + sys_clock_gettime(&proc, &mut host, 0x7fff_ffff), + Err(Errno::EINVAL), + )); + } + + #[test] + fn test_clock_calls_reject_positive_cpu_clock_bit_pattern() { + let proc = Process::new(1); + let mut host = MockHostIO::new(); + + // musl computes 10 for clock_getcpuclockid(-2). The kernel must return + // EINVAL so musl can translate that API result to ESRCH. + assert!(matches!( + sys_clock_gettime(&proc, &mut host, 10), + Err(Errno::EINVAL), + )); + assert!(matches!( + sys_clock_getres(&proc, 10), + Err(Errno::EINVAL), + )); + } + #[test] fn test_nanosleep_rejects_negative() { let proc = Process::new(1); @@ -15857,6 +15949,31 @@ mod tests { assert_eq!(sys_nanosleep(&proc, &mut host, &req), Ok(())); } + #[test] + fn test_clock_nanosleep_boottime_absolute_uses_monotonic_equivalent() { + let proc = Process::new(1); + let mut host = MockHostIO::new(); + let req = WasmTimespec { + tv_sec: 1_234_567_891, + tv_nsec: 123_456_789, + }; + let mut relative = [0_u8; 16]; + + assert_eq!( + sys_clock_nanosleep( + &proc, + &mut host, + wasm_posix_shared::clock::CLOCK_BOOTTIME, + 1, + &req, + relative.as_mut_ptr(), + ), + Ok(()), + ); + assert_eq!(i64::from_le_bytes(relative[0..8].try_into().unwrap()), 1); + assert_eq!(i64::from_le_bytes(relative[8..16].try_into().unwrap()), 0); + } + #[test] fn test_isatty_stdin() { let proc = terminal_process(1); diff --git a/crates/kernel/src/wasm_api.rs b/crates/kernel/src/wasm_api.rs index 16583c446..8af940bef 100644 --- a/crates/kernel/src/wasm_api.rs +++ b/crates/kernel/src/wasm_api.rs @@ -9596,6 +9596,32 @@ pub extern "C" fn kernel_getitimer(which: u32, curr_ptr: *mut u8) -> i32 { /// TIMER_ABSTIME flag for timer_settime. const TIMER_ABSTIME: i32 = 1; +fn timer_clock_to_host_clock(clock_id: u32) -> Option { + use wasm_posix_shared::clock::*; + match clock_id { + CLOCK_REALTIME | CLOCK_MONOTONIC => Some(clock_id), + CLOCK_BOOTTIME => Some(CLOCK_MONOTONIC), + _ => None, + } +} + +#[cfg(test)] +mod posix_timer_tests { + use super::*; + use wasm_posix_shared::clock::*; + + #[test] + fn boottime_timers_use_monotonic_host_clock() { + assert_eq!(timer_clock_to_host_clock(CLOCK_BOOTTIME), Some(CLOCK_MONOTONIC)); + } + + #[test] + fn timer_create_rejects_unsupported_clock_ids() { + assert_eq!(timer_clock_to_host_clock(CLOCK_THREAD_CPUTIME_ID), None); + assert_eq!(timer_clock_to_host_clock(99), None); + } +} + /// timer_create(clock_id, sigevent_ptr, timerid_ptr) /// musl sends ksigevent = {sigev_value(i32), sigev_signo(i32), sigev_notify(i32), sigev_tid(i32)} = 16 bytes. /// Returns 0 on success, negative errno. @@ -9609,6 +9635,11 @@ pub extern "C" fn kernel_timer_create( let (_gkl, proc) = unsafe { get_process() }; + let host_clock_id = match timer_clock_to_host_clock(clock_id) { + Some(id) => id, + None => return -(Errno::EINVAL as i32), + }; + // Parse sigevent (default: SIGEV_SIGNAL with SIGALRM) let (sigev_signo, sigev_value, sigev_notify) = if sevp_ptr.is_null() { (14u32, 0i32, 0u32) // default: SIGALRM with SIGEV_SIGNAL @@ -9645,7 +9676,7 @@ pub extern "C" fn kernel_timer_create( }; proc.posix_timers[timer_id] = Some(PosixTimerState { - clock_id, + clock_id: host_clock_id, sigev_signo, sigev_value, interval_sec: 0, diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index 06f21ee51..510ae8962 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -897,6 +897,9 @@ pub mod clock { pub const CLOCK_MONOTONIC: u32 = 1; pub const CLOCK_PROCESS_CPUTIME_ID: u32 = 2; pub const CLOCK_THREAD_CPUTIME_ID: u32 = 3; + pub const CLOCK_REALTIME_COARSE: u32 = 5; + pub const CLOCK_MONOTONIC_COARSE: u32 = 6; + pub const CLOCK_BOOTTIME: u32 = 7; } /// Timespec structure for the Wasm POSIX interface. diff --git a/docs/architecture.md b/docs/architecture.md index 858ad1b16..4a4642917 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -123,6 +123,37 @@ Key host components: | NodeWorkerAdapter | `worker-adapter.ts` | Creates Node.js worker_threads | | BrowserWorkerAdapter | `worker-adapter-browser.ts` | Creates Web Workers | +### Cooperative process-runtime interrupts + +A process module may import +`env.__wasm_posix_vm_interrupt_after(timed_out_ptr, vm_interrupt_ptr, seconds)` +to request a cooperative runtime deadline. This import is intended for +language runtimes that already poll an interrupt flag at VM safepoints; it is +not a substitute for POSIX signal delivery or a general program-specific +kernel hook. + +The process worker forwards each arm or cancellation request to its host's +dedicated kernel worker. The kernel worker owns the JavaScript timer because a +process worker executing a CPU-bound Wasm loop cannot service its own event +loop. At the monotonic deadline the host sets each flag byte with an atomic +store in the process's shared memory. Timer entries retain the exact +process-generation object as well as the PID, so exec, exit, or PID reuse +cannot redirect a stale callback into a replacement process. Deadlines beyond +JavaScript's signed 32-bit timer range are scheduled in bounded chunks. + +The imported function and its pointer-width-specific signature are part of +the guest/host ABI. A package that starts importing it must raise its kernel +ABI floor in the same ABI reconciliation that versions and snapshots the new +surface. + +Host-runtime lifecycle diagnostics are kept separate from guest file +descriptor 2. Both kernel-worker entries send typed `host_diagnostic` messages; +`NodeKernelHost` and `BrowserKernel` expose them through `onHostDiagnostic`. +Node also records them on the embedding process's console, while the browser +live-demo consumer records them in dmesg. Neither path appends host diagnostics +to the program's stderr byte stream, so test harnesses and applications observe +only bytes the guest actually wrote. + ### 3. Glue Layer (C) **Location**: `libc/glue/` @@ -499,7 +530,20 @@ const ino = mfs.registerLazyFile("/usr/bin/php", "https://cdn.example.com/php.wa await mfs.ensureMaterialized("/usr/bin/php"); ``` -Lazy file metadata (`path`, `url`, `size`, `ino`) can be transferred between instances via `exportLazyEntries()` / `importLazyEntries()` — used when forking workers that share the same SharedArrayBuffer. +Lazy file metadata (`path`, hard-link aliases, `url`, `size`, `ino`, inode +generation, and data-mutation sequence) can be transferred between instances +via `exportLazyEntries()` / `importLazyEntries()` — used when workers share the +same SharedArrayBuffer. The generation prevents an unlinked lazy inode from +transferring its URL or declared size to a later file that reuses the same +guest-visible inode number. The data sequence prevents an asynchronous fetch +from overwriting guest data written through another worker while the request +was in flight. Live cross-worker imports require both identity fields; only a +legacy image whose filesystem bytes and lazy JSON form one trusted artifact can +adopt older metadata, and only for an untouched empty stub. Filesystem rebasing +preserves hard-link identity for lazy and +concrete files rather than copying aliases into independent inodes. A rebase +walks one quiescent source snapshot, so a peer rename cannot mix lazy paths +from one namespace state with bytes from another. ### VFS Images @@ -515,6 +559,17 @@ const image: Uint8Array = await mfs.saveImage(); const fullImage: Uint8Array = await mfs.saveImage({ materializeAll: true }); ``` +Image creation is a quiescent filesystem operation. `saveImage()` rejects a +filesystem with live file or directory descriptors rather than serializing FD +tables, inode open-reference counts, or lock words as durable state. The +resulting bytes contain only filesystem state. Restore also clears those +runtime-only fields in legacy images, so a new machine never inherits handles +or locks from the image builder. Lazy-file and lazy-archive paths are collected +under the same namespace transaction as the filesystem bytes, including names +changed by another worker. `materializeAll: true` resolves both standalone and +archive-backed entries and fails instead of emitting an image that still +depends on a deferred URL. + **Restore from an image:** ```typescript @@ -561,11 +616,10 @@ Build scripts are in `images/vfs/scripts/` and share common helpers (`vfs-image- **Binary format:** -The on-disk file is the raw VFS image below, wrapped in a single zstd -frame. `saveImage()` always writes the compressed form (`.vfs.zst`); -`MemoryFileSystem.fromImage()` accepts either form and auto-detects -the zstd magic (`28 B5 2F FD`) at offset 0 to decide whether to -decompress before parsing. +`MemoryFileSystem.saveImage()` returns the raw VFS image below. The image +builder helper wraps it in one zstd frame for `.vfs.zst` artifacts; +`MemoryFileSystem.fromImage()` accepts either form and auto-detects the zstd +magic (`28 B5 2F FD`) at offset 0 before parsing. Decompressed layout: @@ -573,11 +627,13 @@ Decompressed layout: Offset Size Field 0 4 Magic: 0x56465349 ("VFSI") 4 4 Version: 1 -8 4 Flags: bit 0 = lazy entries included +8 4 Flags: bit 0 = lazy files, bit 1 = lazy archives, bit 2 = metadata 12 4 SharedArrayBuffer data length (N) 16 N Raw SharedArrayBuffer bytes (block filesystem) 16+N 4 Lazy entries JSON length (M) -20+N M Lazy entries as JSON (UTF-8): [{ino, path, url, size}, ...] +20+N M Lazy-file JSON (identity, aliases, URL, declared size) +... 4+L Optional lazy-archive JSON length and bytes (when bit 1 is set) +... 4+P Optional image-metadata JSON length and bytes (when bit 2 is set) ``` ## Networking diff --git a/docs/fork-instrumentation.md b/docs/fork-instrumentation.md index 98c5350de..f45344b32 100644 --- a/docs/fork-instrumentation.md +++ b/docs/fork-instrumentation.md @@ -38,6 +38,10 @@ ABI version: `12` (see - Do not keep compiler/linker flags solely for the retired legacy path. The fork instrumenter does not require preserved function names or onlylists; if a build keeps debug-info flags, it should be for a current diagnostic reason. +- On Unix hosts, the CLI preserves the input Wasm file's permission mode on + its output, including when `--output` names the input file. Package build + scripts can therefore instrument installed executables in place without + making them non-executable. ## State machine diff --git a/docs/posix-status.md b/docs/posix-status.md index 63fc4e13f..245aee249 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -259,7 +259,7 @@ shortcuts. |----------|--------|-------| | `time()` | Full | Wrapper around clock_gettime(CLOCK_REALTIME). Returns seconds since epoch. | | `gettimeofday()` | Full | Wrapper around clock_gettime(CLOCK_REALTIME). Returns (sec, usec) pair. | -| `clock_gettime()` | Full | Host-delegated. CLOCK_REALTIME and CLOCK_MONOTONIC supported. Node.js uses Date.now() and process.hrtime.bigint(). | +| `clock_gettime()` | Partial | Host-delegated. `CLOCK_REALTIME` and `CLOCK_MONOTONIC` are supported. Linux `CLOCK_REALTIME_COARSE` and `CLOCK_MONOTONIC_COARSE` requests use the corresponding host clock as an equivalent fallback because Kandelo hosts do not expose separate coarse sources. `CLOCK_BOOTTIME` is a monotonic-equivalent fallback because Kandelo hosts do not expose suspend accounting. The process/thread CPU clocks currently report elapsed monotonic time rather than authoritative CPU usage. Linux-style encoded process CPU clock IDs must be negative; malformed positive encodings are rejected with `EINVAL` (which musl's `clock_getcpuclockid()` maps to `ESRCH`). Node.js uses `Date.now()` and `process.hrtime.bigint()`; browsers use `Date.now()` and `performance.now()`. | | `nanosleep()` | Partial | Host-delegated. Node.js uses Atomics.wait with timeout. Browser support requires a worker context that can block with Atomics.wait. Validates tv_sec >= 0 and tv_nsec < 1e9. | | `usleep()` | Full | Converts microseconds to sec+nsec, delegates to host_nanosleep. | | `clock_settime()` | Stub | Returns EPERM. Cannot set system clock from Wasm. | @@ -293,7 +293,7 @@ shortcuts. | `inotify_init()` / `inotify_init1()` | Stub | Returns ENOSYS. | | `inotify_add_watch()` / `inotify_rm_watch()` | Stub | Returns EBADF. | | `fanotify_init()` / `fanotify_mark()` | Stub | Returns ENOSYS. | -| `timer_create()` | Partial | CLOCK_REALTIME and CLOCK_MONOTONIC with `SIGEV_SIGNAL` or `SIGEV_NONE`. `SIGEV_THREAD` and Linux-specific `SIGEV_THREAD_ID` return `ENOTSUP`. The timer syscall carries a 32-bit `sival_int`; wasm64 pointer-valued `sigev_value` is not supported. | +| `timer_create()` | Partial | `CLOCK_REALTIME`, `CLOCK_MONOTONIC`, and monotonic-equivalent `CLOCK_BOOTTIME`, with `SIGEV_SIGNAL` or `SIGEV_NONE`. `SIGEV_THREAD` and Linux-specific `SIGEV_THREAD_ID` return `ENOTSUP`. The timer syscall carries a 32-bit `sival_int`; wasm64 pointer-valued `sigev_value` is not supported. | | `timer_settime()` / `timer_gettime()` | Partial | Absolute (TIMER_ABSTIME) and relative timers and automatic interval rearming use host timers with millisecond granularity. `timer_gettime()` and `timer_settime()`'s old-value result currently report the last configured value rather than decreasing remaining time. | | `timer_getoverrun()` | Full | Tracks overrun count when signal is still pending at next interval fire. Reset on successful signal delivery. | | `timer_delete()` | Full | Cancels timer and removes from per-process table. | diff --git a/docs/software-unit-tests.md b/docs/software-unit-tests.md index 11f8a1bfe..8637aaa8a 100644 --- a/docs/software-unit-tests.md +++ b/docs/software-unit-tests.md @@ -64,8 +64,12 @@ scripts/run-php-upstream-node-chunks.sh \ Important reruns included: - `Zend/tests/generators/bug71441.phpt`, which now passes after increasing the - default Node host worker stack from 16 MiB to 32 MiB. This is a general host - stability change for deep guest stacks, not PHP-specific behavior. + Node process-worker stack limit to 32 MiB. This is a general host stability + change for deep guest stacks, not PHP-specific behavior. Embedders with a + tighter memory budget can set `KANDELO_NODE_WORKER_STACK_SIZE_MB` to a + positive finite value; the browser harness instead launches Chromium with + an equivalent V8 stack limit because browser workers do not expose Node's + per-worker `resourceLimits` API. - FPM non-root/virtual-ownership coverage, which now passes after allowing host-backed mounts to expose stable virtual uid/gid metadata to the guest. - Two online `httpbin.org` HTTP/1.1 PHPTs, which are now counted as upstream diff --git a/examples/clock_getcpuclockid_test.c b/examples/clock_getcpuclockid_test.c new file mode 100644 index 000000000..a46c1d153 --- /dev/null +++ b/examples/clock_getcpuclockid_test.c @@ -0,0 +1,34 @@ +#include +#include +#include + +int main(void) { + clockid_t clock_id = 0; + struct timespec ts = {0}; + + /* POSIX returns the error number directly from clock_getcpuclockid(). */ + int rc = clock_getcpuclockid(-2, &clock_id); + if (rc != ESRCH) { + fprintf(stderr, + "FAIL clock_getcpuclockid(-2): rc=%d expected=%d clock=%d\n", + rc, ESRCH, (int)clock_id); + return 1; + } + + errno = 0; + if (clock_gettime((clockid_t)10, &ts) != -1 || errno != EINVAL) { + fprintf(stderr, "FAIL clock_gettime(10): errno=%d expected=%d\n", + errno, EINVAL); + return 1; + } + + errno = 0; + if (clock_getres((clockid_t)10, &ts) != -1 || errno != EINVAL) { + fprintf(stderr, "FAIL clock_getres(10): errno=%d expected=%d\n", + errno, EINVAL); + return 1; + } + + puts("PASS clock id validation"); + return 0; +} diff --git a/examples/getpwent_smoke.c b/examples/getpwent_smoke.c index 9e32c948d..6d01d4ea5 100644 --- a/examples/getpwent_smoke.c +++ b/examples/getpwent_smoke.c @@ -83,6 +83,21 @@ static int check_servbyname(const char *name, const char *proto, int expect_port return port == (unsigned)expect_port ? 0 : 1; } +static int check_service_query(const char *query, const char *proto, + const char *expect_name, int expect_port) { + struct servent *se = getservbyname(query, proto); + if (se == NULL) { + printf("SERVENT query=%s proto=%s result=NULL\n", query, proto); + return 1; + } + unsigned short port = ((unsigned short)se->s_port << 8) | + ((unsigned short)se->s_port >> 8); + printf("SERVENT query=%s name=%s proto=%s port=%u\n", + query, se->s_name, proto, (unsigned)port); + return (strcmp(se->s_name, expect_name) == 0 && + port == (unsigned)expect_port) ? 0 : 1; +} + int main(void) { int rc = 0; rc |= check_pwent_iteration(); @@ -94,6 +109,11 @@ int main(void) { rc |= check_grent_iteration(); rc |= check_servbyname("ssh", "tcp", 22); rc |= check_servbyname("http", "tcp", 80); + rc |= check_service_query("www", "tcp", "www", 80); + rc |= check_service_query("www-http", "tcp", "www-http", 80); + rc |= check_service_query("https", "tcp", "https", 443); + rc |= check_service_query("mysql", "tcp", "mysql", 3306); + rc |= check_service_query("postgresql", "tcp", "postgresql", 5432); printf("DONE rc=%d\n", rc); return rc; } diff --git a/host/src/browser-kernel-host.ts b/host/src/browser-kernel-host.ts index 58fe444dd..45c752a5b 100644 --- a/host/src/browser-kernel-host.ts +++ b/host/src/browser-kernel-host.ts @@ -15,6 +15,7 @@ import { import { FramebufferRegistry } from "./framebuffer/registry"; import type { ProcessSnapshot, SyscallTraceEvent } from "./kernel-worker"; import type { + HostDiagnostic, MainToKernelMessage, KernelToMainMessage, } from "./browser-kernel-protocol"; @@ -49,6 +50,8 @@ export interface BrowserKernelOptions { onStdout?: (data: Uint8Array) => void; /** Called when a process writes to stderr */ onStderr?: (data: Uint8Array) => void; + /** Called for host-runtime diagnostics that are not guest stderr. */ + onHostDiagnostic?: (diagnostic: HostDiagnostic) => void; /** Called when a process requests a TCP listener (for service worker bridging) */ onListenTcp?: (pid: number, fd: number, port: number) => void; /** Called when the service-worker HTTP bridge gains or completes preview requests. */ @@ -85,6 +88,10 @@ export interface BrowserKernelOptions { syscallLogPtrWidth?: 4 | 8; /** Forwarded to TlsNetworkBackendOptions.dnsAliases. */ dnsAliases?: Record; + /** Forwarded to TlsNetworkBackendOptions.corsProxyUrl. Browser pages that + * are not controlled by Kandelo's service worker can use this to route + * guest outbound HTTP(S) through a same-origin proxy. */ + corsProxyUrl?: string; } /** Options for {@link BrowserKernel.boot}. */ @@ -329,13 +336,26 @@ export class BrowserKernel { this.handleWorkerMessage(e.data as KernelToMainMessage); }; this.kernelWorkerHandle.onerror = (e: ErrorEvent) => { - console.error("[BrowserKernel] Kernel worker error:", e.message); const err = new Error(`Kernel worker error: ${e.message}`); for (const [, { reject }] of this.pendingRequests) { reject(err); } this.pendingRequests.clear(); this.options.onHttpBridgePendingRequests?.(0); + const diagnostic: HostDiagnostic = { + pid: 0, + source: "kernel worker", + message: `[BrowserKernel] kernel worker error: ${e.message}`, + }; + // A worker-level error cannot send a typed message itself. Preserve the + // same callback contract and a visible default without treating the + // failure as guest stderr. + console.error(diagnostic.message); + try { + this.options.onHostDiagnostic?.(diagnostic); + } catch (callbackError) { + console.error("[BrowserKernel] onHostDiagnostic callback failed:", callbackError); + } }; await new Promise((resolve, reject) => { @@ -393,6 +413,7 @@ export class BrowserKernel { enableSyscallLog: this.options.enableSyscallLog, syscallLogPtrWidth: this.options.syscallLogPtrWidth, dnsAliases: this.options.dnsAliases, + corsProxyUrl: this.options.corsProxyUrl, }, }; this.kernelWorkerHandle.postMessage(initMsg, [transferBuf]); @@ -803,14 +824,18 @@ export class BrowserKernel { /** * Register lazy files: creates stubs in the VFS and forwards metadata - * to the kernel worker so it can materialize on demand via sync XHR. + * to the kernel worker so it can materialize asynchronously before use. */ registerLazyFiles(entries: Array<{ path: string; url: string; size: number; mode?: number }>): void { const fs = this.fs; const lazyEntries: LazyFileEntry[] = []; for (const e of entries) { - const ino = fs.registerLazyFile(e.path, e.url, e.size, e.mode); - lazyEntries.push({ ino, path: e.path, url: e.url, size: e.size }); + fs.registerLazyFile(e.path, e.url, e.size, e.mode); + const entry = fs.getLazyEntry(e.path); + if (!entry) { + throw new Error(`Failed to register lazy VFS file: ${e.path}`); + } + lazyEntries.push(entry); } const requestId = this.nextRequestId++; void this.request(requestId, { @@ -1033,6 +1058,12 @@ export class BrowserKernel { private handleWorkerMessage(msg: KernelToMainMessage): void { switch (msg.type) { + case "ready": + case "init_error": + // The temporary boot listener resolves or rejects initialization. The + // permanent listener also receives these messages, so account for + // them explicitly rather than relying on an implicit fall-through. + break; case "response": { const pending = this.pendingRequests.get(msg.requestId); if (pending) { @@ -1069,6 +1100,15 @@ export class BrowserKernel { case "stderr": this.options.onStderr?.(msg.data); break; + case "host_diagnostic": { + this.options.onHostDiagnostic?.({ + pid: msg.pid, + source: msg.source, + message: msg.message, + ...(msg.status === undefined ? {} : { status: msg.status }), + }); + break; + } case "pty_output": { const cb = this.ptyOutputCallbacks.get(msg.pid); if (cb) { @@ -1115,6 +1155,17 @@ export class BrowserKernel { case "lazy_download": this.emitLazyDownload(msg.event); break; + default: { + // Keep this dispatch coupled to KernelToMainMessage as the protocol + // grows. Runtime values still originate outside TypeScript, so make a + // malformed/unknown worker message visible instead of dropping it. + const exhaustive: never = msg; + void exhaustive; + console.error( + `[BrowserKernel] unknown kernel-worker message type: ${String((msg as { type?: unknown }).type)}`, + ); + break; + } } } diff --git a/host/src/browser-kernel-protocol.ts b/host/src/browser-kernel-protocol.ts index b64d202cb..5ef1f6c8c 100644 --- a/host/src/browser-kernel-protocol.ts +++ b/host/src/browser-kernel-protocol.ts @@ -9,8 +9,12 @@ import type { HttpResponse, } from "./networking/in-kernel-http"; import type { LazyDownloadEvent } from "./vfs/memory-fs"; +import type { + HostDiagnosticMessage, +} from "./host-diagnostic"; export type { HttpRequest, HttpResponse }; +export type { HostDiagnostic } from "./host-diagnostic"; // ── Main Thread → Kernel Worker ── @@ -59,6 +63,10 @@ export interface InitMessage { syscallLogPtrWidth?: 4 | 8; /** Forwarded to TlsNetworkBackendOptions.dnsAliases. */ dnsAliases?: Record; + /** Forwarded to TlsNetworkBackendOptions.corsProxyUrl for browser fetch + * backends that need a same-origin proxy to reach external HTTP(S) + * hosts. */ + corsProxyUrl?: string; }; } @@ -494,6 +502,7 @@ export type KernelToMainMessage = | ExitMessage | StdoutMessage | StderrMessage + | HostDiagnosticMessage | PtyOutputMessage | ListenTcpMessage | FbBindMessage diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index 155f19489..292229385 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -96,6 +96,7 @@ import { removeThreadWorkerRegistryEntry, threadWorkerFailureDisposition, } from "./thread-worker-disposition"; +import { VmInterruptTimerManager } from "./vm-interrupt-timer"; import type { CentralizedWorkerInitMessage, CentralizedThreadInitMessage, @@ -111,6 +112,7 @@ import { type ProcessMemoryLayout, } from "./process-memory"; import type { + HostDiagnostic, MainToKernelMessage, KernelToMainMessage, } from "./browser-kernel-protocol"; @@ -151,6 +153,9 @@ interface ProcessInfo { } const processes = new Map(); const processTeardowns = new Map>(); +const vmInterruptTimers = new VmInterruptTimerManager( + (pid) => processes.get(pid), +); // Includes standalone thread-worker teardown promises that may outlive the // process map entry they came from. const workerTeardowns = new Set>(); @@ -194,7 +199,18 @@ async function resolveExecutableForLaunch( const shebang = parseShebang(bytes); if (!shebang) { if (!isWasmModuleBytes(bytes)) return { errno: ENOEXEC }; - return { programBytes: bytes, argv }; + let programModule: WebAssembly.Module; + try { + programModule = await WebAssembly.compile(bytes); + } catch (error) { + if (error instanceof WebAssembly.CompileError) return { errno: ENOEXEC }; + throw error; + } + const declaredAbi = extractAbiVersion(bytes); + if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { + return { errno: ENOEXEC }; + } + return { programBytes: bytes, programModule, argv }; } const scriptArgv = [ @@ -220,6 +236,16 @@ const threadWorkers = new Map(); const threadExits = new ThreadExitCoordinator(); const reportedNonzeroProcessExits = new Set(); +function handleVmInterruptTimer(msg: { + pid: number; + timedOutPtr: number; + vmInterruptPtr: number; + seconds: number; +}, pid: number, process: ProcessInfo): void { + if (msg.pid !== pid) return; + vmInterruptTimers.handleRequest(pid, process, msg); +} + function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -253,8 +279,12 @@ function reportNonzeroProcessExitDiagnostic( `[kernel-worker] nonzero process exit pid=${pid} status=${status} source=${source} argv=${JSON.stringify(info?.argv ?? [])}` + (serviceLog ? `\n${serviceLog}` : "") + `\n${syscalls}`; - console.warn(diagnostic); - post({ type: "stderr", pid, data: new TextEncoder().encode(`${diagnostic}\n`) }); + reportHostDiagnostic({ + pid, + status, + source, + message: diagnostic, + }, "warn"); } function readServiceLogForProcess(argv: readonly string[] | undefined): string | null { @@ -321,6 +351,15 @@ function post(msg: KernelToMainMessage, transfer?: Transferable[]) { (globalThis as any).postMessage(msg, transfer ?? []); } +function reportHostDiagnostic( + diagnostic: HostDiagnostic, + level: "error" | "warn" = "error", +): void { + if (level === "warn") console.warn(diagnostic.message); + else console.error(diagnostic.message); + post({ type: "host_diagnostic", ...diagnostic }); +} + function reportBridgePendingRequests(): void { post({ type: "http_bridge_pending", count: activeBridgeRequests.size }); } @@ -377,11 +416,10 @@ function respondErrorIfRequested( } function reportWorkerProtocolError(message: string): void { - console.error(`[kernel-worker] ${message}`); - post({ - type: "stderr", + reportHostDiagnostic({ pid: 0, - data: new TextEncoder().encode(`[kernel-worker] ${message}\n`), + source: "worker protocol", + message: `[kernel-worker] ${message}`, }); } @@ -698,6 +736,7 @@ async function handleInit(msg: Extract) { // production, keeping the browser networking path identical across modes. const tlsBackend = new TlsNetworkBackend({ dnsAliases: msg.config.dnsAliases, + corsProxyUrl: msg.config.corsProxyUrl, }); await tlsBackend.init(); io.network = tlsBackend; @@ -994,8 +1033,6 @@ function installProcessWorkerListeners( const message = `[kernel-worker] pid=${pid} ${source} -> forcing exit ${status}`; if (status === 0 && source === "worker-main exit message") { console.debug(message); - } else { - console.warn(message); } reportNonzeroProcessExitDiagnostic(pid, status, source); handleExit(pid, status, crashSignum, worker); @@ -1009,9 +1046,15 @@ function installProcessWorkerListeners( // m.status — worker-main posted {type:"exit"}, normal exit path. worker.on("error", (err: Error) => { if (intentionallyTerminated.has(worker as object)) return; - console.error(`[kernel-worker] Worker error pid=${pid}:`, err.message); const signum = classifiedSignalOrFallback(err); - finalize(signalExitStatus(signum), "worker.onerror", signum); + const status = signalExitStatus(signum); + reportHostDiagnostic({ + pid, + status, + source: "worker.onerror", + message: `[kernel-worker] worker error pid=${pid}: ${err.message}`, + }); + finalize(status, "worker.onerror", signum); }); worker.on("exit", (code: number) => { // BrowserWorkerHandle synthesizes an "exit" event when the underlying @@ -1029,24 +1072,23 @@ function installProcessWorkerListeners( }); worker.on("message", (msg: unknown) => { if (intentionallyTerminated.has(worker as object)) return; - if (processes.get(pid)?.worker !== worker) return; - const m = msg as { type?: string; message?: string; pid?: number; status?: number }; + const process = processes.get(pid); + if (!process || process.worker !== worker) return; + const m = msg as WorkerToHostMessage; if (m.type === "error") { - console.error(`[kernel-worker] Process error pid=${pid}:`, m.message); - // Forward to host stderr so the demo log shows the actual failure - // ("Kernel worker failed: ..." with the wasm trap or - // instantiation error). Without this, a process death in the - // browser is invisible to the user — only console.error in the - // kernel-worker scope, which most users don't open. Mirrors the - // Node-side handleSpawn message-listener stderr forwarding. - const errBytes = new TextEncoder().encode( - `[process-worker] ${m.message ?? "unknown error"}\n`, - ); - post({ type: "stderr", pid, data: errBytes }); const signum = classifiedSignalOrFallback(m.message); - finalize(classifiedTrapExitStatus(m.message) ?? -1, "worker-main error message", signum); + const status = classifiedTrapExitStatus(m.message) ?? -1; + reportHostDiagnostic({ + pid, + status, + source: "worker-main error message", + message: `[process-worker] ${m.message ?? "unknown error"}`, + }); + finalize(status, "worker-main error message", signum); } else if (m.type === "exit") { finalize(m.status ?? 0, "worker-main exit message"); + } else if (m.type === "vm_interrupt_timer") { + handleVmInterruptTimer(m, pid, process); } }); } @@ -1148,17 +1190,7 @@ async function handleExec( const resolved = await resolveExecutableForLaunch(path, argv); if (!resolved) return -2; // ENOENT if ("errno" in resolved) return -resolved.errno; - const { programBytes: bytes, argv: launchArgv } = resolved; - - try { - await WebAssembly.compile(bytes); - } catch { - return -8; // ENOEXEC: reject malformed modules before changing old state - } - const declaredAbi = extractAbiVersion(bytes); - if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { - return -8; - } + const { programBytes: bytes, programModule, argv: launchArgv } = resolved; // Preallocate the replacement address space before the irreversible commit. const ptrWidth = detectPtrWidth(bytes); const metadataResult = kernelWorker.validateExecMetadata( @@ -1192,6 +1224,7 @@ async function handleExec( try { const setupResult = kernelWorker.kernelExecSetup(pid, callerTid); if (setupResult < 0) return setupResult; + vmInterruptTimers.clear(pid); // Invalidate the discarded image synchronously at the commit point. This // keeps stale clone/listener continuations out even if later detach or @@ -1232,6 +1265,7 @@ async function handleExec( pid, ppid: 0, programBytes: bytes, + programModule, memory: newMemory, channelOffset: newChannelOffset, argv: launchArgv, @@ -1259,6 +1293,7 @@ async function handleExec( processes.set(pid, { memory: newMemory, programBytes: bytes, + programModule, worker: replacementWorker, argv: launchArgv, channelOffset: newChannelOffset, @@ -1289,10 +1324,11 @@ async function handleExec( const message = err instanceof Error ? err.message : String(err); try { - post({ - type: "stderr", + reportHostDiagnostic({ pid, - data: new TextEncoder().encode(`[exec] post-commit transition failed: ${message}\n`), + status: signalExitStatus(SIGSEGV), + source: "exec post-commit transition", + message: `[exec] post-commit transition failed: ${message}`, }); } catch { // A closed host port must not prevent kernel-side reap. @@ -1311,9 +1347,9 @@ async function handleExec( * Handle SYS_SPAWN (non-forking posix_spawn) on the browser host. * * The kernel has already constructed the child Process descriptor under - * `childPid` with attrs and file actions applied. This callback resolves - * `path` to bytes via the shared MemoryFileSystem, allocates a fresh - * Memory for the child, registers it with the kernel + * `childPid` with attrs and file actions applied. This callback receives the + * preflight's compiled program, allocates a fresh Memory for the child, and + * registers it with the kernel * (`skipKernelCreate: true` — kernel did its half), and spawns a Worker. * * Distinct from handleExec (which replaces the calling worker) and @@ -1328,9 +1364,9 @@ async function handleExec( /** * Pre-flight resolver — see node-kernel-worker-entry.ts:handlePosixSpawnResolve. * Browser-side equivalent: materialize the lazy file (async fetch via - * the memfs lazy-loader, avoiding sync-XHR + SW deadlocks) then read its - * contents from the VFS. Side-effect-free; safe to call on PATH search - * iterations that may not resolve. + * the memfs lazy-loader, avoiding sync-XHR + SW deadlocks), reads its + * contents from the VFS, follows shebangs, and compiles the final Wasm + * module. Safe to call before the kernel applies spawn file actions. */ async function handlePosixSpawnResolve( path: string, @@ -1340,25 +1376,23 @@ async function handlePosixSpawnResolve( } /** - * Launch a worker for a SYS_SPAWN child whose program bytes have already - * been resolved by `handlePosixSpawnResolve`. Mirrors the Node entry's - * `handlePosixSpawn`. + * Launch a worker for a SYS_SPAWN child whose program has already been + * resolved and compiled by `handlePosixSpawnResolve`. Mirrors the Node + * entry's `handlePosixSpawn`. */ async function handlePosixSpawn( childPid: number, - programBytes: ArrayBuffer, - argv: string[], + program: ResolvedSpawnProgram, envp: string[], ): Promise { await waitForProcessTeardowns(); - // The child can receive a group-directed signal while the browser waits for - // an unrelated teardown. Keep the successful spawn and kernel-owned zombie, - // but never install a Worker for a process that has already terminated. + // Unrelated teardown waits yield to the event loop. Keep a successfully + // created zombie, but never resurrect it with a new Worker. if (!kernelWorker.shouldLaunchPendingChild(childPid)) return 0; - post({ type: "proc_event", kind: "spawn", pid: childPid }); + const { programBytes, programModule, argv } = program; const ptrWidth = detectPtrWidth(programBytes); const { memory: newMemory, @@ -1385,6 +1419,7 @@ async function handlePosixSpawn( pid: childPid, ppid: 0, programBytes, + programModule, memory: newMemory, channelOffset: newChannelOffset, argv, @@ -1398,6 +1433,7 @@ async function handlePosixSpawn( processes.set(childPid, { memory: newMemory, programBytes, + programModule, worker: newWorker, argv, channelOffset: newChannelOffset, @@ -1457,10 +1493,10 @@ async function handleClone( alloc = processInfo.threadAllocator.allocate(memory); } catch (e) { const message = e instanceof Error ? e.message : String(e); - post({ - type: "stderr", + reportHostDiagnostic({ pid, - data: new TextEncoder().encode(`[kernel-worker] pid=${pid}: ${message}\n`), + source: "clone allocation", + message: `[kernel-worker] pid=${pid}: ${message}`, }); throw e; } @@ -1541,9 +1577,15 @@ async function handleClone( void terminateThreadEntry(); return; } - const text = `[kernel-worker] pid=${pid} tid=${tid}: ${reason}\n`; - post({ type: "stderr", pid, data: new TextEncoder().encode(text) }); const disposition = threadWorkerFailureDisposition(reason); + reportHostDiagnostic({ + pid, + status: disposition.kind === "guest-fatal-trap" + ? disposition.exitStatus + : undefined, + source: "thread worker failure", + message: `[kernel-worker] pid=${pid} tid=${tid}: ${reason}`, + }); kernelWorker.finalizeThreadExit(pid, tid, alloc.channelOffset); void terminateThreadEntry(); if (disposition.kind === "guest-fatal-trap") { @@ -1563,10 +1605,12 @@ async function handleClone( // worker-main posted {type:"error"} — instantiation failure, top-level // throw, etc. Without this the parent's pthread_join blocks forever. failThread((m as { message?: string }).message ?? "thread error"); + } else if (m.type === "vm_interrupt_timer") { + if (!isCurrentThreadGeneration() || m.pid !== pid) return; + handleVmInterruptTimer(m, pid, processInfo); } }); threadWorker.on("error", (err: Error) => { - console.error(`[kernel-worker] thread worker error pid=${pid} tid=${tid}:`, err.message); failThread(`worker error: ${err.message ?? err}`); }); @@ -1600,6 +1644,7 @@ async function finishProcessExit( const info = processes.get(pid); if (!info || info.worker !== expectedWorker) return; if (processTeardowns.has(expectedWorker)) return; + vmInterruptTimers.clear(pid); reportNonzeroProcessExitDiagnostic(pid, exitStatus, "kernel process exit"); const threadedSettleMs = threadedProcessPids.has(pid) @@ -1660,6 +1705,7 @@ async function finishProcessExit( async function handleTerminateProcess(msg: Extract) { const pid = msg.pid; + vmInterruptTimers.clear(pid); // Terminate thread workers const threads = threadWorkers.get(pid); @@ -1834,6 +1880,7 @@ async function handleDestroy(msg: Extract { kernelWorker.attachKmsStats(msg.crtcId, msg.stats); break; default: { + // Every typed MainToKernelMessage must have a case above. Browser + // tooling also sends a few deliberately out-of-band control messages, + // which remain handled below after the compile-time exhaustiveness + // check. + const exhaustive: never = msg; + void exhaustive; // Handle non-protocol messages (e.g., bridge port transfer) const raw = e.data as any; if (raw?.type === "sysprof_start") { @@ -2167,7 +2220,11 @@ sw.onmessage = (e: MessageEvent) => { }; } } else { - console.warn("[kernel-worker] Unknown message type:", raw?.type); + reportHostDiagnostic({ + pid: 0, + source: "worker protocol", + message: `[kernel-worker] unknown main-thread message type: ${String(raw?.type)}`, + }, "warn"); } } } diff --git a/host/src/browser.ts b/host/src/browser.ts index c6d8597ea..e294063fd 100644 --- a/host/src/browser.ts +++ b/host/src/browser.ts @@ -11,6 +11,7 @@ export { centralizedWorkerMain, centralizedThreadWorkerMain, patchWasmForThread export type { MessagePort as WorkerMessagePort } from "./worker-main"; export type { KernelConfig, PlatformIO, StatResult, StatfsResult } from "./types"; export type { WorkerAdapter, WorkerHandle } from "./worker-adapter"; +export type { HostDiagnostic } from "./host-diagnostic"; export type { HostToWorkerMessage, WorkerToHostMessage, WorkerReadyMessage, WorkerExitMessage, WorkerErrorMessage, diff --git a/host/src/dylink.ts b/host/src/dylink.ts index 1bd43deee..5904ed89e 100644 --- a/host/src/dylink.ts +++ b/host/src/dylink.ts @@ -324,6 +324,14 @@ export interface LoadSharedLibraryOptions { got: Map; /** Already-loaded libraries for dedup and dependency resolution */ loadedLibraries: Map; + /** + * Process-owned exception tag shared by the main image and every side + * module. When omitted, standalone linker users get one tag lazily and it + * is retained on this options object for subsequent loads. + */ + longjmpTag?: WebAssembly.Tag; + /** Process pointer width, which also determines the __c_longjmp payload. */ + ptrWidth?: 4 | 8; /** Immutable symbol names exported by the main module. */ mainModuleSymbols?: ReadonlySet; /** Present only in a process worker that can drive side-module unwind. */ @@ -336,6 +344,56 @@ export interface LoadSharedLibraryOptions { resolveLibrarySync?: (name: string) => Uint8Array | null; } +type TagConstructor = new ( + descriptor: { parameters: Array<"i32" | "i64"> }, +) => WebAssembly.Tag; + +function tagConstructor(): TagConstructor | undefined { + return (WebAssembly as typeof WebAssembly & { Tag?: TagConstructor }).Tag; +} + +/** Create the exception tag used by one process and all of its side modules. */ +export function createLongjmpTag(ptrWidth: 4 | 8): WebAssembly.Tag | undefined { + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new TypeError(`invalid process pointer width ${String(ptrWidth)}`); + } + const Tag = tagConstructor(); + return Tag + ? new Tag({ parameters: [ptrWidth === 8 ? "i64" : "i32"] }) + : undefined; +} + +/** Reject lookalike values before handing an exception-tag import to Wasm. */ +export function requireLongjmpTag(value: unknown, context: string): WebAssembly.Tag { + const Tag = tagConstructor(); + if (!Tag) { + throw new Error(`${context}: this WebAssembly runtime does not support exception tags`); + } + if (!(value instanceof Tag)) { + throw new TypeError(`${context}: __c_longjmp must be an actual WebAssembly.Tag`); + } + return value; +} + +function validateLongjmpConfiguration(options: LoadSharedLibraryOptions): void { + const ptrWidth = options.ptrWidth ?? 4; + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new TypeError(`invalid process pointer width ${String(ptrWidth)}`); + } + if (options.longjmpTag !== undefined) { + requireLongjmpTag(options.longjmpTag, "dynamic linker"); + } +} + +function resolveLongjmpTag(options: LoadSharedLibraryOptions): WebAssembly.Tag { + validateLongjmpConfiguration(options); + if (options.longjmpTag !== undefined) return options.longjmpTag; + const ptrWidth = options.ptrWidth ?? 4; + const tag = createLongjmpTag(ptrWidth); + options.longjmpTag = requireLongjmpTag(tag, "dynamic linker"); + return options.longjmpTag; +} + const SIDE_DYNAMIC_LOOKUP_IMPORTS = new Set([ "__wasm_dlopen", "__wasm_dlsym", @@ -447,6 +505,12 @@ function instantiateSharedLibrary( && imp.kind === "function" && SIDE_DYNAMIC_LOOKUP_IMPORTS.has(imp.name) ); + const importsLongjmpTag = moduleImports.some((imp) => + imp.module === "env" + && imp.name === "__c_longjmp" + && (imp.kind as string) === "tag" + ); + const longjmpTag = importsLongjmpTag ? resolveLongjmpTag(options) : undefined; if (presentForkExports.length > 0 && !hasCompleteForkInstrumentation) { const missing = SIDE_MODULE_FORK_EXPORTS.filter((exportName) => @@ -643,16 +707,6 @@ function instantiateSharedLibrary( return entry; }; - // Tag imported by side modules compiled with clang's wasm SjLj lowering - // (`-mllvm -wasm-enable-sjlj`). The host doesn't actually catch these — the - // main process either has its own __c_longjmp tag (LLVM 22) or doesn't use - // SjLj (LLVM 21). A stub Tag lets the side module's import type-check and - // instantiate; behavior at throw time is undefined but the side module - // typically never throws this tag itself. - const longjmpTag = (typeof (WebAssembly as any).Tag === "function") - ? new (WebAssembly as any).Tag({ parameters: ["i32"] }) - : undefined; - let instance: WebAssembly.Instance | null = null; let sideForkState: SideModuleForkState | null = null; const forkState = (): number => { @@ -857,6 +911,7 @@ export async function loadSharedLibrary( wasmBytes: Uint8Array, options: LoadSharedLibraryOptions, ): Promise { + validateLongjmpConfiguration(options); const existing = options.loadedLibraries.get(name); if (existing) return existing; @@ -891,6 +946,7 @@ export function loadSharedLibrarySync( options: LoadSharedLibraryOptions, replay?: DylinkReplayOptions, ): LoadedSharedLibrary { + validateLongjmpConfiguration(options); const existing = options.loadedLibraries.get(name); if (existing) return existing; @@ -938,6 +994,7 @@ export class DynamicLinker { private lastError: string | null = null; constructor(options: LoadSharedLibraryOptions) { + validateLongjmpConfiguration(options); this.options = options; } diff --git a/host/src/host-diagnostic.ts b/host/src/host-diagnostic.ts new file mode 100644 index 000000000..1b267b39b --- /dev/null +++ b/host/src/host-diagnostic.ts @@ -0,0 +1,16 @@ +/** + * Host-runtime diagnostic delivered outside a guest process's fd 2 stream. + * + * `status` is present when the diagnostic describes a process disposition. + * Protocol/setup failures that are not tied to an exit status leave it absent. + */ +export interface HostDiagnostic { + pid: number; + status?: number; + source: string; + message: string; +} + +export interface HostDiagnosticMessage extends HostDiagnostic { + type: "host_diagnostic"; +} diff --git a/host/src/index.ts b/host/src/index.ts index a65bc990b..c8164c40e 100644 --- a/host/src/index.ts +++ b/host/src/index.ts @@ -91,6 +91,7 @@ export { WasiShim, WasiExit } from "./wasi-shim"; export { isWasiModule, wasiModuleImportsMemory, wasiModuleDefinesMemory } from "./wasi-detect"; export { NodeKernelHost } from "./node-kernel-host"; export type { NodeKernelHostOptions, SpawnOptions } from "./node-kernel-host"; +export type { HostDiagnostic } from "./host-diagnostic"; export type { MainToKernelMessage, KernelToMainMessage, diff --git a/host/src/kernel-worker.ts b/host/src/kernel-worker.ts index 6144bd213..a24610e60 100644 --- a/host/src/kernel-worker.ts +++ b/host/src/kernel-worker.ts @@ -118,10 +118,12 @@ const E2BIG = 7; const EAGAIN = 11; const EACCES = 13; const EBADF = 9; +const EADDRNOTAVAIL = 99; const EEXIST = 17; const EFAULT = 14; const EIO = 5; const EINVAL = 22; +const ENODEV = 19; const ENOMEM = 12; const ENAMETOOLONG = 36; const ENOENT = 2; @@ -204,9 +206,19 @@ const SIGCHLD = 17; const SIGALRM = 14; /** Network ioctl request codes */ +const SIOCGIFNAME = 0x8910; const SIOCGIFCONF = 0x8912; const SIOCGIFHWADDR = 0x8927; const SIOCGIFADDR = 0x8915; +const SIOCGIFINDEX = 0x8933; +const AF_INET = 2; +const ARPHRD_ETHER = 1; +const ARPHRD_LOOPBACK = 772; +const IF_NAMESIZE = 16; +const VIRTUAL_INTERFACES = [ + { name: "lo", index: 1, loopback: true }, + { name: "eth0", index: 2, loopback: false }, +] as const; /** Ioctl syscall number */ const SYS_IOCTL = ABI_SYSCALLS.Ioctl; @@ -700,6 +712,7 @@ export interface ForkFromThreadContext { export interface ResolvedSpawnProgram { programBytes: ArrayBuffer; + programModule: WebAssembly.Module; argv: string[]; } @@ -707,13 +720,12 @@ export interface SpawnResolveError { errno: number; } -export type SpawnProgramResolution = ArrayBuffer | ResolvedSpawnProgram | SpawnResolveError; +export type SpawnProgramResolution = ResolvedSpawnProgram | SpawnResolveError; function isSpawnResolveError( resolution: SpawnProgramResolution, ): resolution is SpawnResolveError { - return !(resolution instanceof ArrayBuffer) && - "errno" in resolution && + return "errno" in resolution && typeof resolution.errno === "number"; } @@ -762,10 +774,10 @@ export interface CentralizedKernelCallbacks { ) => Promise; /** - * Pre-flight resolution step for SYS_SPAWN. Returns the program bytes - * for `path` (or `{ programBytes, argv }` when resolution rewrites argv, - * e.g. a shebang script), `{ errno }` for a located but unlaunchable - * program, or `null` for ENOENT. **Must NOT have side effects** — + * Pre-flight resolution step for SYS_SPAWN. Returns the validated program + * bytes, their compiled module, and launch argv for `path`, `{ errno }` for + * a located but unlaunchable program, or `null` for ENOENT. **Must NOT have + * side effects** — * `handleSpawn` calls this BEFORE `kernel_spawn_process` so that file * actions never run on a doomed PATH-iteration. POSIX requires * file_actions to run "exactly once," and `posix_spawnp`'s PATH-walk @@ -779,11 +791,12 @@ export interface CentralizedKernelCallbacks { onResolveSpawn?: (path: string, argv: string[]) => Promise; /** - * Launch a worker for the spawned child with already-resolved bytes - * and argv (from `onResolveSpawn`). The kernel has constructed the child Process - * descriptor under `childPid` and applied file actions + attrs by the - * time this is called. The callback instantiates a fresh Worker and - * registers it via `registerProcess({ skipKernelCreate: true })`. + * Launch a worker for the spawned child with the already-resolved bytes, + * compiled module, and argv from `onResolveSpawn`. The kernel has + * constructed the child Process descriptor under `childPid` and applied + * file actions + attrs by the time this is called. The callback instantiates + * a fresh Worker and registers it via + * `registerProcess({ skipKernelCreate: true })`. * * Returns 0 on success, negative errno on failure. On non-zero return * the kernel descriptor is rolled back via `kernel_remove_process`. @@ -792,7 +805,11 @@ export interface CentralizedKernelCallbacks { * `onFork` (which clones the parent's Memory): `onSpawn` always * creates a fresh Memory and runs the new program from `_start`. */ - onSpawn?: (childPid: number, programBytes: ArrayBuffer, argv: string[], envp: string[]) => Promise; + onSpawn?: ( + childPid: number, + program: ResolvedSpawnProgram, + envp: string[], + ) => Promise; /** * Called when a process calls clone (thread creation). The callback should @@ -3051,7 +3068,7 @@ export class CentralizedKernelWorker { return; } - // --- ioctl: intercept network interface ioctls (SIOCGIFCONF, SIOCGIFHWADDR) --- + // --- ioctl: intercept network interface ioctls --- // These require host-side handling because: // SIOCGIFCONF: struct ifconf contains a pointer to a process-memory buffer // SIOCGIFHWADDR: returns the virtual MAC address for this kernel instance @@ -3061,6 +3078,10 @@ export class CentralizedKernelWorker { this.handleIoctlIfconf(channel, origArgs); return; } + if (request === SIOCGIFNAME) { + this.handleIoctlIfname(channel, origArgs); + return; + } if (request === SIOCGIFHWADDR) { this.handleIoctlIfhwaddr(channel, origArgs); return; @@ -3069,6 +3090,10 @@ export class CentralizedKernelWorker { this.handleIoctlIfaddr(channel, origArgs); return; } + if (request === SIOCGIFINDEX) { + this.handleIoctlIfindex(channel, origArgs); + return; + } } // --- fcntl with struct flock pointer --- @@ -6057,8 +6082,64 @@ export class CentralizedKernelWorker { } // ---- Network interface ioctl host-side handlers ---- - // The kernel has a single virtual network interface ("eth0") with a random - // MAC address generated per kernel instance. + + private finishNetworkIoctl( + channel: ChannelInfo, + retVal = 0, + errno = 0, + ): void { + this.completeChannelRaw(channel, retVal, errno); + this.relistenChannel(channel); + } + + private guestRangeIsValid( + channel: ChannelInfo, + ptr: number, + length: number, + ): boolean { + return Number.isSafeInteger(ptr) && + Number.isSafeInteger(length) && + ptr >= 0 && + length >= 0 && + ptr <= channel.memory.buffer.byteLength - length; + } + + private interfaceAddress( + iface: (typeof VIRTUAL_INTERFACES)[number], + ): Uint8Array | null { + if (iface.loopback) return new Uint8Array([127, 0, 0, 1]); + const address = this.io.network?.localAddress; + return address?.length === 4 ? new Uint8Array(address) : null; + } + + /** + * `struct ifreq` has a 16-byte name followed by a union. The union is 16 + * bytes under wasm32, but its `struct ifmap` member grows to 24 bytes under + * wasm64 because `unsigned long` is pointer-sized. + */ + private ifreqSize(channel: ChannelInfo): number { + return this.getPtrWidth(channel.pid) === 8 ? 40 : 32; + } + + private readIfreqName(channel: ChannelInfo, ifreqPtr: number): string | null { + if (!this.guestRangeIsValid(channel, ifreqPtr, this.ifreqSize(channel))) { + return null; + } + const bytes = new Uint8Array(channel.memory.buffer, ifreqPtr, IF_NAMESIZE); + let end = 0; + while (end < bytes.length && bytes[end] !== 0) end++; + return new TextDecoder().decode(new Uint8Array(bytes.subarray(0, end))); + } + + private writeIfreqName( + processMem: Uint8Array, + ifreqPtr: number, + name: string, + ): void { + const nameBytes = new TextEncoder().encode(name); + processMem.fill(0, ifreqPtr, ifreqPtr + IF_NAMESIZE); + processMem.set(nameBytes.subarray(0, IF_NAMESIZE - 1), ifreqPtr); + } /** * Handle SIOCGIFCONF: enumerate network interfaces. @@ -6067,16 +6148,22 @@ export class CentralizedKernelWorker { * directly — we handle the entire ioctl on the host side. */ private handleIoctlIfconf(channel: ChannelInfo, origArgs: number[]): void { - const processView = new DataView(channel.memory.buffer); - const processMem = new Uint8Array(channel.memory.buffer); const pw = this.getPtrWidth(channel.pid); - - // Read struct ifconf from process memory at arg[2] - // struct ifconf { int ifc_len; union { char *ifc_buf; struct ifreq *ifc_req; }; } - // wasm32: ifc_len at +0 (4B), ifc_buf at +4 (4B) — total 8 bytes - // wasm64: ifc_len at +0 (4B), [4B padding], ifc_buf at +8 (8B) — total 16 bytes const ifconfPtr = origArgs[2]; + const ifconfSize = pw === 8 ? 16 : 8; + if (!this.guestRangeIsValid(channel, ifconfPtr, ifconfSize)) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + + const processView = new DataView(channel.memory.buffer); + const processMem = new Uint8Array(channel.memory.buffer); + const ifreqSize = this.ifreqSize(channel); const ifcLen = processView.getInt32(ifconfPtr, true); + if (ifcLen < 0) { + this.finishNetworkIoctl(channel, -EINVAL, EINVAL); + return; + } let ifcBuf: number; if (pw === 8) { ifcBuf = Number(processView.getBigUint64(ifconfPtr + 8, true)); @@ -6084,32 +6171,65 @@ export class CentralizedKernelWorker { ifcBuf = processView.getUint32(ifconfPtr + 4, true); } - // sizeof(struct ifreq) = 32 (16 name + 16 sockaddr union, no pointers — same on both) - const SIZEOF_IFREQ = 32; + if (ifcBuf === 0) { + processView.setInt32( + ifconfPtr, + VIRTUAL_INTERFACES.length * ifreqSize, + true, + ); + this.finishNetworkIoctl(channel); + return; + } - if (ifcLen >= SIZEOF_IFREQ && ifcBuf !== 0) { - // Write one ifreq entry for "eth0" into process memory at ifc_buf - const nameBytes = new TextEncoder().encode("eth0"); - processMem.set(nameBytes, ifcBuf); - processMem.fill(0, ifcBuf + nameBytes.length, ifcBuf + 16); // pad ifr_name + if (ifcLen < ifreqSize) { + processView.setInt32(ifconfPtr, 0, true); + this.finishNetworkIoctl(channel); + return; + } - // ifr_addr: AF_INET (2) with 127.0.0.1 — just needs to be a valid sockaddr - processMem.fill(0, ifcBuf + 16, ifcBuf + SIZEOF_IFREQ); - processView.setUint16(ifcBuf + 16, 2, true); // AF_INET - processMem[ifcBuf + 20] = 127; // sin_addr = 127.0.0.1 - processMem[ifcBuf + 21] = 0; - processMem[ifcBuf + 22] = 0; - processMem[ifcBuf + 23] = 1; + const capacity = Math.floor(ifcLen / ifreqSize); + const count = Math.min(capacity, VIRTUAL_INTERFACES.length); + const bytesToWrite = count * ifreqSize; + if (!this.guestRangeIsValid(channel, ifcBuf, bytesToWrite)) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } - // Update ifc_len to actual bytes written - processView.setInt32(ifconfPtr, SIZEOF_IFREQ, true); - } else { - // No space or null buffer - processView.setInt32(ifconfPtr, 0, true); + for (let i = 0; i < count; i++) { + const iface = VIRTUAL_INTERFACES[i]; + const entryPtr = ifcBuf + i * ifreqSize; + this.writeIfreqName(processMem, entryPtr, iface.name); + processMem.fill(0, entryPtr + IF_NAMESIZE, entryPtr + ifreqSize); + processView.setUint16(entryPtr + IF_NAMESIZE, AF_INET, true); + const address = this.interfaceAddress(iface); + if (address) processMem.set(address, entryPtr + IF_NAMESIZE + 4); } + processView.setInt32(ifconfPtr, bytesToWrite, true); + this.finishNetworkIoctl(channel); + } - this.completeChannelRaw(channel, 0, 0); - this.relistenChannel(channel); + /** + * Handle SIOCGIFNAME: map an interface index to its name. + * struct ifreq at arg[2]: ifr_name[16] + union; ifr_ifindex lives at +16. + */ + private handleIoctlIfname(channel: ChannelInfo, origArgs: number[]): void { + const ifreqPtr = origArgs[2]; + if (!this.guestRangeIsValid(channel, ifreqPtr, this.ifreqSize(channel))) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + const processView = new DataView(channel.memory.buffer); + const processMem = new Uint8Array(channel.memory.buffer); + const ifindex = processView.getInt32(ifreqPtr + 16, true); + const iface = VIRTUAL_INTERFACES.find((candidate) => candidate.index === ifindex); + + if (!iface) { + this.finishNetworkIoctl(channel, -ENODEV, ENODEV); + return; + } + + this.writeIfreqName(processMem, ifreqPtr, iface.name); + this.finishNetworkIoctl(channel); } /** @@ -6118,41 +6238,97 @@ export class CentralizedKernelWorker { * Returns the virtual MAC in ifr_hwaddr.sa_data[0..5]. */ private handleIoctlIfhwaddr(channel: ChannelInfo, origArgs: number[]): void { + const ifreqPtr = origArgs[2]; + const name = this.readIfreqName(channel, ifreqPtr); + if (name === null) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + const iface = VIRTUAL_INTERFACES.find((candidate) => candidate.name === name); + if (!iface) { + this.finishNetworkIoctl(channel, -ENODEV, ENODEV); + return; + } const processView = new DataView(channel.memory.buffer); const processMem = new Uint8Array(channel.memory.buffer); - const ifreqPtr = origArgs[2]; - // Write ifr_hwaddr at offset 16 from ifreq start: - // sa_family = ARPHRD_ETHER (1) - // sa_data[0..5] = MAC address - processMem.fill(0, ifreqPtr + 16, ifreqPtr + 32); // clear sockaddr - processView.setUint16(ifreqPtr + 16, 1, true); // ARPHRD_ETHER - processMem.set(this.virtualMacAddress, ifreqPtr + 18); // MAC address + processMem.fill( + 0, + ifreqPtr + IF_NAMESIZE, + ifreqPtr + this.ifreqSize(channel), + ); + processView.setUint16( + ifreqPtr + IF_NAMESIZE, + iface.loopback ? ARPHRD_LOOPBACK : ARPHRD_ETHER, + true, + ); + if (!iface.loopback) { + processMem.set(this.virtualMacAddress, ifreqPtr + IF_NAMESIZE + 2); + } - this.completeChannelRaw(channel, 0, 0); - this.relistenChannel(channel); + this.finishNetworkIoctl(channel); } /** * Handle SIOCGIFADDR: get interface address. * struct ifreq at arg[2]: ifr_name[16] + ifr_addr (struct sockaddr, 16 bytes) - * Returns 127.0.0.1 for the virtual interface. + * Returns the selected virtual interface's assigned IPv4 address. */ private handleIoctlIfaddr(channel: ChannelInfo, origArgs: number[]): void { + const ifreqPtr = origArgs[2]; + const name = this.readIfreqName(channel, ifreqPtr); + if (name === null) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + const iface = VIRTUAL_INTERFACES.find((candidate) => candidate.name === name); + if (!iface) { + this.finishNetworkIoctl(channel, -ENODEV, ENODEV); + return; + } + const address = this.interfaceAddress(iface); + if (!address) { + this.finishNetworkIoctl(channel, -EADDRNOTAVAIL, EADDRNOTAVAIL); + return; + } const processView = new DataView(channel.memory.buffer); const processMem = new Uint8Array(channel.memory.buffer); + + processMem.fill( + 0, + ifreqPtr + IF_NAMESIZE, + ifreqPtr + this.ifreqSize(channel), + ); + processView.setUint16(ifreqPtr + IF_NAMESIZE, AF_INET, true); + processMem.set(address, ifreqPtr + IF_NAMESIZE + 4); + + this.finishNetworkIoctl(channel); + } + + /** + * Handle SIOCGIFINDEX: map an interface name to its index. + * struct ifreq at arg[2]: ifr_name[16] + union; ifr_ifindex lives at +16. + */ + private handleIoctlIfindex(channel: ChannelInfo, origArgs: number[]): void { const ifreqPtr = origArgs[2]; + const name = this.readIfreqName(channel, ifreqPtr); + if (name === null) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + const iface = VIRTUAL_INTERFACES.find((candidate) => candidate.name === name); - // Write ifr_addr at offset 16: AF_INET + 127.0.0.1 - processMem.fill(0, ifreqPtr + 16, ifreqPtr + 32); - processView.setUint16(ifreqPtr + 16, 2, true); // AF_INET - processMem[ifreqPtr + 20] = 127; // sin_addr = 127.0.0.1 - processMem[ifreqPtr + 21] = 0; - processMem[ifreqPtr + 22] = 0; - processMem[ifreqPtr + 23] = 1; + if (!iface) { + this.finishNetworkIoctl(channel, -ENODEV, ENODEV); + return; + } - this.completeChannelRaw(channel, 0, 0); - this.relistenChannel(channel); + new DataView(channel.memory.buffer).setInt32( + ifreqPtr + IF_NAMESIZE, + iface.index, + true, + ); + this.finishNetworkIoctl(channel); } private handleWritev(channel: ChannelInfo, syscallNr: number, origArgs: number[]): void { @@ -7260,14 +7436,14 @@ export class CentralizedKernelWorker { return; } - // ── PRE-FLIGHT: resolve program bytes BEFORE calling the kernel ── + // ── PRE-FLIGHT: resolve and compile BEFORE calling the kernel ── // POSIX requires file_actions to run "exactly once." `posix_spawnp`'s // PATH search emits one `posix_spawn` per candidate; if we let the // kernel apply file_actions on each iteration, the side effects // (e.g. `addopen(O_EXCL)`) accumulate and the second iteration sees // its own state from the first. Resolve bytes via the host's // side-effect-free preflight first; only call the kernel if the - // program actually exists. + // program actually exists and compiles. const resolveSpawnProgram = async (): Promise => { const resolved = await this.callbacks.onResolveSpawn!(path, argv); if (resolved || rawPath === path || !rawPath || rawPath.startsWith("/")) { @@ -7292,10 +7468,8 @@ export class CentralizedKernelWorker { this.completeChannel(channel, SYS_SPAWN, origArgs, undefined, -1, resolved.errno >>> 0); return; } - const programBytes = resolved instanceof ArrayBuffer ? resolved : resolved.programBytes; - const launchArgv = resolved instanceof ArrayBuffer ? argv : resolved.argv; this.handleSpawnAfterResolve( - channel, origArgs, parentPid, pidOutPtr, blobBytes, blobLen, launchArgv, envp, programBytes, + channel, origArgs, parentPid, pidOutPtr, blobBytes, blobLen, resolved, envp, ); }).catch((err) => { if (!this.isAsyncChannelProcessActive(channel)) return; @@ -7306,8 +7480,8 @@ export class CentralizedKernelWorker { /** * Continuation of `handleSpawn` after `onResolveSpawn` has returned - * actual program bytes. Now safe to ask the kernel to build the child - * (which will apply file_actions exactly once). + * validated, compiled program. Now safe to ask the kernel to build the + * child (which will apply file_actions exactly once). */ private handleSpawnAfterResolve( channel: ChannelInfo, @@ -7316,9 +7490,8 @@ export class CentralizedKernelWorker { pidOutPtr: number, blobBytes: Uint8Array, blobLen: number, - argv: string[], + program: ResolvedSpawnProgram, envp: string[], - programBytes: ArrayBuffer, ): void { // ── Copy blob to kernel scratch ── const kernelMem = new Uint8Array(this.kernelMemory!.buffer); @@ -7367,14 +7540,14 @@ export class CentralizedKernelWorker { try { this.inheritHostFdMirrors(parentPid, childPid, false); launch = Promise.resolve( - this.callbacks.onSpawn!(childPid, programBytes, argv, envp), + this.callbacks.onSpawn!(childPid, program, envp), ); } catch (err) { rollbackSpawn(5, err); return; } - // ── Launch the worker async (with already-resolved bytes) ── + // ── Launch the worker async (with the precompiled program) ── launch.then((rc) => { if (rc < 0) { rollbackSpawn((-rc) >>> 0); diff --git a/host/src/networking/fetch-backend.ts b/host/src/networking/fetch-backend.ts index 83b126855..9100e16b3 100644 --- a/host/src/networking/fetch-backend.ts +++ b/host/src/networking/fetch-backend.ts @@ -1,5 +1,8 @@ import type { NetworkIO } from "../types"; -import { parseNumericIpv4Hostname, validateDnsHostname } from "./hostname"; +import { + parseNumericIpv4Hostname, + validateSyntheticDnsHostname, +} from "./hostname"; /** Error with errno property for EAGAIN propagation to the kernel host imports. */ export class EagainError extends Error { @@ -11,6 +14,7 @@ const POLLIN = 0x0001; const POLLOUT = 0x0004; const POLLERR = 0x0008; const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; interface ConnectionState { hostname: string; @@ -172,7 +176,7 @@ export class FetchNetworkBackend implements NetworkIO { return data.length; } - recv(handle: number, maxLen: number, _flags: number): Uint8Array { + recv(handle: number, maxLen: number, flags: number): Uint8Array { const conn = this.connections.get(handle); if (!conn) throw new Error("ENOTCONN"); @@ -193,7 +197,9 @@ export class FetchNetworkBackend implements NetworkIO { if (len === 0) return new Uint8Array(0); const result = conn.responseBuf.slice(conn.responseOffset, conn.responseOffset + len); - conn.responseOffset += len; + if ((flags & MSG_PEEK) === 0) { + conn.responseOffset += len; + } return result; } @@ -230,7 +236,7 @@ export class FetchNetworkBackend implements NetworkIO { getaddrinfo(hostname: string): Uint8Array { const literalIp = parseNumericIpv4Hostname(hostname); if (literalIp) return literalIp; - validateDnsHostname(hostname); + validateSyntheticDnsHostname(hostname, this.options.hostAliases); // In the browser, return a synthetic IP. // The actual connection uses the Host header, not this IP. diff --git a/host/src/networking/hostname.ts b/host/src/networking/hostname.ts index a6de88654..bb46dcf04 100644 --- a/host/src/networking/hostname.ts +++ b/host/src/networking/hostname.ts @@ -72,3 +72,28 @@ export function validateDnsHostname(hostname: string): void { throw nameNotFoundError(hostname); } } + +/** + * Validate a name before a browser fetch/TLS backend assigns it a synthetic + * address. These backends defer the real lookup to fetch(), so they must reject + * names that the browser environment already knows cannot resolve. + */ +export function validateSyntheticDnsHostname( + hostname: string, + aliases?: Record, +): void { + validateDnsHostname(hostname); + + const absoluteName = hostname.endsWith(".") ? hostname.slice(0, -1) : hostname; + const lowerName = absoluteName.toLowerCase(); + if ( + aliases && + (Object.prototype.hasOwnProperty.call(aliases, absoluteName) || + Object.prototype.hasOwnProperty.call(aliases, lowerName)) + ) { + return; + } + if (lowerName === "invalid" || lowerName.endsWith(".invalid")) { + throw nameNotFoundError(hostname); + } +} diff --git a/host/src/networking/tcp-backend.ts b/host/src/networking/tcp-backend.ts index dfa996fb9..cd69e2030 100644 --- a/host/src/networking/tcp-backend.ts +++ b/host/src/networking/tcp-backend.ts @@ -8,6 +8,7 @@ const POLLIN = 0x0001; const POLLOUT = 0x0004; const POLLERR = 0x0008; const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; /** * Map a Node.js network error code to a POSIX errno value. @@ -138,7 +139,7 @@ export class TcpNetworkBackend implements NetworkIO { return data.length; } - recv(handle: number, maxLen: number, _flags: number): Uint8Array { + recv(handle: number, maxLen: number, flags: number): Uint8Array { const conn = this.connections.get(handle); if (!conn) throw new Error("ENOTCONN"); if (conn.error) throw conn.error; @@ -150,7 +151,9 @@ export class TcpNetworkBackend implements NetworkIO { conn.recvBuf.byteOffset, len, ); - conn.recvBuf = conn.recvBuf.subarray(len); + if ((flags & MSG_PEEK) === 0) { + conn.recvBuf = conn.recvBuf.subarray(len); + } return result; } diff --git a/host/src/networking/tls-network-backend.ts b/host/src/networking/tls-network-backend.ts index aabf9dde9..fe4817234 100644 --- a/host/src/networking/tls-network-backend.ts +++ b/host/src/networking/tls-network-backend.ts @@ -15,7 +15,10 @@ import type { NetworkIO } from "../types"; import { EagainError } from "./fetch-backend"; -import { parseNumericIpv4Hostname, validateDnsHostname } from "./hostname"; +import { + parseNumericIpv4Hostname, + validateSyntheticDnsHostname, +} from "./hostname"; import { TLS_1_2_Connection } from "../../../packages/registry/openssl/src/tls/1_2/connection"; import { generateCertificate, @@ -23,6 +26,12 @@ import { type GeneratedCertificate, } from "../../../packages/registry/openssl/src/tls/certificates"; +const POLLIN = 0x0001; +const POLLOUT = 0x0004; +const POLLERR = 0x0008; +const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; + // ------------------------------------------------------------------ types interface HttpConnectionState { @@ -232,7 +241,7 @@ export class TlsNetworkBackend implements NetworkIO { getaddrinfo(hostname: string): Uint8Array { const literalIp = parseNumericIpv4Hostname(hostname); if (literalIp) return literalIp; - validateDnsHostname(hostname); + validateSyntheticDnsHostname(hostname, this.dnsAliases); const ip = this.syntheticIp(hostname); const ipStr = this.ipKey(ip); @@ -275,14 +284,14 @@ export class TlsNetworkBackend implements NetworkIO { return this.httpSend(conn, data); } - recv(handle: number, maxLen: number, _flags: number): Uint8Array { + recv(handle: number, maxLen: number, flags: number): Uint8Array { const conn = this.connections.get(handle); if (!conn) throw new Error("ENOTCONN"); if (conn.kind === "tls") { - return this.tlsRecv(conn, maxLen); + return this.tlsRecv(conn, maxLen, flags); } - return this.httpRecv(conn, maxLen); + return this.httpRecv(conn, maxLen, flags); } close(handle: number): void { @@ -422,14 +431,16 @@ export class TlsNetworkBackend implements NetworkIO { return data.length; } - private tlsRecv(conn: TlsConnectionState, maxLen: number): Uint8Array { + private tlsRecv(conn: TlsConnectionState, maxLen: number, flags: number): Uint8Array { if (conn.error) throw conn.error; // Check if we have encrypted data buffered from the TLS engine if (conn.clientDownstreamBuf.length > 0) { const n = Math.min(maxLen, conn.clientDownstreamBuf.length); const result = conn.clientDownstreamBuf.slice(0, n); - conn.clientDownstreamBuf = conn.clientDownstreamBuf.subarray(n); + if ((flags & MSG_PEEK) === 0) { + conn.clientDownstreamBuf = conn.clientDownstreamBuf.subarray(n); + } return result; } @@ -620,7 +631,7 @@ export class TlsNetworkBackend implements NetworkIO { return data.length; } - private httpRecv(conn: HttpConnectionState, maxLen: number): Uint8Array { + private httpRecv(conn: HttpConnectionState, maxLen: number, flags: number): Uint8Array { if (!conn.fetchDone) { throw new EagainError(); } @@ -636,10 +647,50 @@ export class TlsNetworkBackend implements NetworkIO { if (len === 0) return new Uint8Array(0); const result = conn.responseBuf.slice(conn.responseOffset, conn.responseOffset + len); - conn.responseOffset += len; + if ((flags & MSG_PEEK) === 0) { + conn.responseOffset += len; + } return result; } + poll(handle: number, events: number): number { + const conn = this.connections.get(handle); + if (!conn) throw Object.assign(new Error("ENOTCONN"), { errno: 107 }); + + let revents = 0; + if ((events & POLLOUT) !== 0 && (conn.kind === "http" || !conn.closed)) { + revents |= POLLOUT; + } + + if (conn.kind === "http") { + if (conn.fetchError) return revents | POLLERR; + if ( + (events & POLLIN) !== 0 && + conn.responseBuf && + conn.responseOffset < conn.responseBuf.length + ) { + revents |= POLLIN; + } + if ( + conn.fetchDone && + conn.responseBuf && + conn.responseOffset >= conn.responseBuf.length + ) { + revents |= POLLHUP; + } + return revents; + } + + if (conn.error) return revents | POLLERR; + if ((events & POLLIN) !== 0 && conn.clientDownstreamBuf.length > 0) { + revents |= POLLIN; + } + if (conn.closed && conn.clientDownstreamBuf.length === 0) { + revents |= POLLHUP; + } + return revents; + } + // ---- Utilities ---- private syntheticIp(hostname: string): Uint8Array { diff --git a/host/src/networking/virtual-network.ts b/host/src/networking/virtual-network.ts index 1e5b54cce..e59f74294 100644 --- a/host/src/networking/virtual-network.ts +++ b/host/src/networking/virtual-network.ts @@ -21,6 +21,7 @@ const POLLIN = 0x0001; const POLLOUT = 0x0004; const POLLERR = 0x0008; const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; const ANY = "0.0.0.0"; @@ -95,7 +96,7 @@ class VirtualTcpPeer implements TcpConnectionPeer { return data.length; } - recv(maxLen: number, _flags: number): Uint8Array { + recv(maxLen: number, flags: number): Uint8Array { if (this.reset) { const err = new Error("ECONNRESET") as Error & { errno?: number }; err.errno = ECONNRESET; @@ -104,7 +105,9 @@ class VirtualTcpPeer implements TcpConnectionPeer { if (this.recvBuf.length > 0) { const len = Math.min(maxLen, this.recvBuf.length); const out = this.recvBuf.slice(0, len); - this.recvBuf = this.recvBuf.slice(len); + if ((flags & MSG_PEEK) === 0) { + this.recvBuf = this.recvBuf.slice(len); + } return out; } if (!this.peer || this.peer.writeClosed) { diff --git a/host/src/node-kernel-host.ts b/host/src/node-kernel-host.ts index a2fe31f68..c97fc1a87 100644 --- a/host/src/node-kernel-host.ts +++ b/host/src/node-kernel-host.ts @@ -20,6 +20,7 @@ import { createRequire } from "node:module"; import { Worker as NodeThreadWorker } from "node:worker_threads"; import { resolveBinary } from "./binary-resolver"; import type { + HostDiagnostic, MainToKernelMessage, KernelToMainMessage, ResolveExecRequestMessage, @@ -61,6 +62,8 @@ export interface NodeKernelHostOptions { onStdout?: (pid: number, data: Uint8Array) => void; /** Called when a process writes to stderr */ onStderr?: (pid: number, data: Uint8Array) => void; + /** Called for host-runtime diagnostics that are not guest stderr. */ + onHostDiagnostic?: (diagnostic: HostDiagnostic) => void; /** Called when a process writes PTY output */ onPtyOutput?: (pid: number, data: Uint8Array) => void; /** Called when a process is spawned, execs a new program, or exits. @@ -88,7 +91,15 @@ export interface NodeKernelHostOptions { * to a VFS-only world yet. */ rootfsImage?: "default" | ArrayBuffer | Uint8Array; - extraMounts?: Array<{ mountPoint: string; hostPath: string; readonly?: boolean }>; + extraMounts?: Array<{ + mountPoint: string; + hostPath: string; + readonly?: boolean; + /** Virtual owner for existing host-backed mount entries. Defaults to root. */ + uid?: number; + /** Virtual group for existing host-backed mount entries. Defaults to root. */ + gid?: number; + }>; } export interface SpawnOptions { @@ -144,6 +155,20 @@ export class NodeKernelHost { reject(error); } this.pendingRequests.clear(); + const diagnostic: HostDiagnostic = { + pid: 0, + source: "kernel worker", + message: `[NodeKernelHost] kernel worker error: ${error.message}`, + }; + // A worker-level error cannot send a typed message itself. Preserve the + // same callback contract and a visible default without treating the + // failure as guest stderr. + console.error(diagnostic.message); + try { + this.options.onHostDiagnostic?.(diagnostic); + } catch (callbackError) { + console.error("[NodeKernelHost] onHostDiagnostic callback failed:", callbackError); + } }); // Send init and wait for ready @@ -480,6 +505,10 @@ export class NodeKernelHost { private handleWorkerMessage(msg: KernelToMainMessage): void { switch (msg.type) { + case "ready": + // The temporary init listener resolves readiness. The permanent + // listener also receives the message, so account for it explicitly. + break; case "response": { const pending = this.pendingRequests.get(msg.requestId); if (pending) { @@ -524,12 +553,32 @@ export class NodeKernelHost { case "stderr": this.options.onStderr?.(msg.pid, msg.data); break; + case "host_diagnostic": { + this.options.onHostDiagnostic?.({ + pid: msg.pid, + source: msg.source, + message: msg.message, + ...(msg.status === undefined ? {} : { status: msg.status }), + }); + break; + } case "pty_output": this.options.onPtyOutput?.(msg.pid, msg.data); break; case "resolve_exec": this.handleResolveExec(msg); break; + default: { + // Keep this dispatch coupled to KernelToMainMessage as the protocol + // grows. Runtime values still originate outside TypeScript, so make a + // malformed/unknown worker message visible instead of dropping it. + const exhaustive: never = msg; + void exhaustive; + console.error( + `[NodeKernelHost] unknown kernel-worker message type: ${String((msg as { type?: unknown }).type)}`, + ); + break; + } } } diff --git a/host/src/node-kernel-protocol.ts b/host/src/node-kernel-protocol.ts index 853fc19b4..c1e5145e1 100644 --- a/host/src/node-kernel-protocol.ts +++ b/host/src/node-kernel-protocol.ts @@ -11,8 +11,10 @@ * See docs/plans/2026-04-30-external-kernel-http-request-interface.md. */ import type { HttpRequest, HttpResponse } from "./networking/in-kernel-http"; +import type { HostDiagnosticMessage } from "./host-diagnostic"; export type { HttpRequest, HttpResponse }; +export type { HostDiagnostic } from "./host-diagnostic"; // ── Main Thread → Kernel Worker ── @@ -37,7 +39,13 @@ export interface InitMessage { * (custom-io / legacy path). */ rootfsImage?: ArrayBuffer; - extraMounts?: Array<{ mountPoint: string; hostPath: string; readonly?: boolean }>; + extraMounts?: Array<{ + mountPoint: string; + hostPath: string; + readonly?: boolean; + uid?: number; + gid?: number; + }>; /** Attach a real-TCP backend (TcpNetworkBackend) to the worker's PlatformIO * so wasm programs can dial external hosts via Node `net.Socket`. */ enableTcpNetwork?: boolean; @@ -260,6 +268,7 @@ export type KernelToMainMessage = | ExitMessage | StdoutMessage | StderrMessage + | HostDiagnosticMessage | PtyOutputMessage | ResolveExecRequestMessage | ProcEventMessage; diff --git a/host/src/node-kernel-worker-entry.ts b/host/src/node-kernel-worker-entry.ts index 3ce9eeb1f..fb2689511 100644 --- a/host/src/node-kernel-worker-entry.ts +++ b/host/src/node-kernel-worker-entry.ts @@ -10,8 +10,8 @@ * Main → Worker: init, spawn, append_stdin_data, set_stdin_data, * pty_write, pty_resize, terminate_process, destroy, * resolve_exec_response - * Worker → Main: ready, response, exit, stdout, stderr, pty_output, - * resolve_exec + * Worker → Main: ready, response, exit, stdout, stderr, host_diagnostic, + * pty_output, resolve_exec */ import { parentPort } from "node:worker_threads"; import { readFileSync, existsSync, mkdtempSync, rmSync } from "node:fs"; @@ -59,6 +59,7 @@ import { removeThreadWorkerRegistryEntry, threadWorkerFailureDisposition, } from "./thread-worker-disposition"; +import { VmInterruptTimerManager } from "./vm-interrupt-timer"; import { computeProcessMemoryLayout, createProcessMemory, @@ -73,6 +74,7 @@ import type { WorkerToHostMessage, } from "./worker-protocol"; import type { + HostDiagnostic, MainToKernelMessage, KernelToMainMessage, InitMessage, @@ -116,7 +118,11 @@ interface ProcessInfo { } const processes = new Map(); const processTeardowns = new Map>(); +const vmInterruptTimers = new VmInterruptTimerManager( + (pid) => processes.get(pid), +); const reportedExits = new Set(); +const reportedNonzeroProcessExits = new Set(); // Workers terminated by the kernel-worker entry itself (handleExit / // handleExec / handleTerminate). The crash safety-net listener checks @@ -142,11 +148,16 @@ function installCrashSafetyNet( if (intentionallyTerminated.has(worker as object)) return; const cur = processes.get(pid); if (!cur || cur.worker !== worker) return; // already torn down or replaced - const errBytes = new TextEncoder().encode( - `[process-worker] pid=${pid} crashed (worker exit code=${code}, no SYS_exit_group from wasm)\n`, - ); - post({ type: "stderr", pid, data: errBytes }); - void finalizeProcessWorker(pid, worker, signalExitStatus(SIGSEGV), SIGSEGV); + const status = signalExitStatus(SIGSEGV); + reportHostDiagnostic({ + pid, + status, + source: "worker exit event", + message: + `[process-worker] pid=${pid} crashed ` + + `(worker exit code=${code}, no SYS_exit_group from wasm)`, + }); + void finalizeProcessWorker(pid, worker, status, SIGSEGV); }); } @@ -193,9 +204,28 @@ async function terminateThreadWorkers(pid: number): Promise { function reportProcessExit(pid: number, status: number): void { if (reportedExits.has(pid)) return; reportedExits.add(pid); + if (status !== 0 && !reportedNonzeroProcessExits.has(pid)) { + reportedNonzeroProcessExits.add(pid); + reportHostDiagnostic({ + pid, + status, + source: "process exit", + message: `[node-kernel-worker] nonzero process exit pid=${pid} status=${status}`, + }, "warn"); + } post({ type: "exit", pid, status }); } +function handleVmInterruptTimer(msg: { + pid: number; + timedOutPtr: number; + vmInterruptPtr: number; + seconds: number; +}, pid: number, process: ProcessInfo): void { + if (msg.pid !== pid) return; + vmInterruptTimers.handleRequest(pid, process, msg); +} + function signalFromExitStatus(exitStatus: number): number | null { return exitStatus >= 128 ? (exitStatus - 128) & 0x7f : null; } @@ -233,6 +263,7 @@ async function finalizeProcessWorker( if (intentionallyTerminated.has(worker as object)) return; const cur = processes.get(pid); if (!cur || cur.worker !== worker) return; + vmInterruptTimers.clear(pid); // Synthesize a signal-style reap *before* `deactivateProcess` in // case the worker died without sending SYS_EXIT_GROUP (uncaught @@ -280,9 +311,13 @@ function finalizeProcessWorkerError( ): void { if (intentionallyTerminated.has(worker as object)) return; if (processes.get(pid)?.worker !== worker) return; - const errBytes = new TextEncoder().encode(`[process-worker] ${message ?? "unknown error"}\n`); - post({ type: "stderr", pid, data: errBytes }); const { exitStatus, signum } = processWorkerErrorDisposition(message); + reportHostDiagnostic({ + pid, + status: exitStatus, + source: "worker-main error message", + message: `[process-worker] ${message ?? "unknown error"}`, + }); void finalizeProcessWorker(pid, worker, exitStatus, signum); } @@ -295,9 +330,13 @@ function finalizeUnexpectedWorkerError( if (intentionallyTerminated.has(worker as object)) return; if (processes.get(pid)?.worker !== worker) return; const message = err instanceof Error ? (err.message ?? String(err)) : String(err); - const errBytes = new TextEncoder().encode(`[kernel-worker] pid=${pid}: ${label}: ${message}\n`); - post({ type: "stderr", pid, data: errBytes }); const { exitStatus, signum } = unexpectedWorkerCrashDisposition(err); + reportHostDiagnostic({ + pid, + status: exitStatus, + source: label, + message: `[kernel-worker] pid=${pid}: ${label}: ${message}`, + }); void finalizeProcessWorker(pid, worker, exitStatus, signum); } @@ -305,6 +344,23 @@ function post(msg: KernelToMainMessage) { port.postMessage(msg); } +function reportHostDiagnostic( + diagnostic: HostDiagnostic, + level: "error" | "warn" = "error", +): void { + if (level === "warn") console.warn(diagnostic.message); + else console.error(diagnostic.message); + post({ type: "host_diagnostic", ...diagnostic }); +} + +function reportWorkerProtocolError(message: string): void { + reportHostDiagnostic({ + pid: 0, + source: "worker protocol", + message: `[node-kernel-worker] ${message}`, + }); +} + function respond(requestId: number, result: unknown) { post({ type: "response", requestId, result }); } @@ -501,7 +557,18 @@ async function resolveExecutableForLaunch( const shebang = parseShebang(bytes); if (!shebang) { if (!isWasmModuleBytes(bytes)) return { errno: ENOEXEC }; - return { programBytes: bytes, argv }; + let programModule: WebAssembly.Module; + try { + programModule = await WebAssembly.compile(bytes); + } catch (error) { + if (error instanceof WebAssembly.CompileError) return { errno: ENOEXEC }; + throw error; + } + const declaredAbi = extractAbiVersion(bytes); + if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { + return { errno: ENOEXEC }; + } + return { programBytes: bytes, programModule, argv }; } const scriptArgv = [ @@ -553,7 +620,13 @@ function installDefaultCaBundle(fs: MemoryFileSystem): void { */ function buildVirtualPlatformIO( rootfsImage: ArrayBuffer, - extraMounts?: Array<{ mountPoint: string; hostPath: string; readonly?: boolean }>, + extraMounts?: Array<{ + mountPoint: string; + hostPath: string; + readonly?: boolean; + uid?: number; + gid?: number; + }>, ): VirtualPlatformIO { const bootSessionDir = mkdtempSync(join(tmpdir(), "wasm-posix-session-")); sessionDir = bootSessionDir; @@ -567,7 +640,10 @@ function buildVirtualPlatformIO( shmfs.chmod("/", 0o1777); const extras: MountConfig[] = (extraMounts ?? []).map((m) => ({ mountPoint: m.mountPoint, - backend: new HostFileSystem(m.hostPath), + backend: new HostFileSystem(m.hostPath, m.mountPoint, { + uid: m.uid, + gid: m.gid, + }), readonly: m.readonly, })); const mounts = [ @@ -761,7 +837,7 @@ function handleSpawn(msg: SpawnMessage) { // — so surface them to stderr and synthesize an exit so the host's // exitResolver fires with a non-zero status. worker.on("message", (raw: unknown) => { - const m = raw as { type: string; pid?: number; message?: string; status?: number }; + const m = raw as WorkerToHostMessage; if (m.type === "error" && m.pid === pid) { finalizeProcessWorkerError(pid, worker, m.message); } else if (m.type === "exit" && m.pid === pid) { @@ -770,6 +846,9 @@ function handleSpawn(msg: SpawnMessage) { // the kernel didn't process a SYS_exit_group first, the kernel // still has the process registered and host.spawn() would hang. void finalizeProcessWorker(pid, worker, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === pid) { + const process = processes.get(pid); + if (process?.worker === worker) handleVmInterruptTimer(m, pid, process); } }); @@ -858,11 +937,16 @@ async function handleFork( childWorker.on("error", (err: Error) => finalizeUnexpectedWorkerError(childPid, childWorker, "worker error", err)); childWorker.on("message", (raw: unknown) => { - const m = raw as { type: string; pid?: number; message?: string; status?: number }; + const m = raw as WorkerToHostMessage; if (m.type === "error" && m.pid === childPid) { finalizeProcessWorkerError(childPid, childWorker, m.message); } else if (m.type === "exit" && m.pid === childPid) { void finalizeProcessWorker(childPid, childWorker, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === childPid) { + const process = processes.get(childPid); + if (process?.worker === childWorker) { + handleVmInterruptTimer(m, childPid, process); + } } }); @@ -885,17 +969,7 @@ async function handleExec( const resolved = await resolveExecutableForLaunch(path, argv); if (!resolved) return -2; // ENOENT if ("errno" in resolved) return -resolved.errno; - const { programBytes, argv: launchArgv } = resolved; - - try { - await WebAssembly.compile(programBytes); - } catch { - return -8; // ENOEXEC: reject malformed modules before changing old state - } - const declaredAbi = extractAbiVersion(programBytes); - if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { - return -8; // ENOEXEC: known ABI mismatch is a truthful launch failure - } + const { programBytes, programModule, argv: launchArgv } = resolved; const newPtrWidth = detectPtrWidth(programBytes); const metadataResult = kernelWorker.validateExecMetadata( launchArgv, @@ -924,6 +998,7 @@ async function handleExec( try { const setupResult = kernelWorker.kernelExecSetup(pid, callerTid); if (setupResult < 0) return setupResult; + vmInterruptTimers.clear(pid); // From this point onward the old image cannot resume. Invalidate its // channels and async continuations immediately, before any other @@ -956,6 +1031,7 @@ async function handleExec( pid, ppid: 0, programBytes, + programModule, memory: newMemory, channelOffset: newChannelOffset, argv: launchArgv, @@ -984,6 +1060,7 @@ async function handleExec( processes.set(pid, { memory: newMemory, programBytes, + programModule, worker: replacementWorker, channelOffset: newChannelOffset, ptrWidth: newPtrWidth, @@ -998,11 +1075,16 @@ async function handleExec( // uncaught wasm traps) so the host learns the process died — same // wiring as handleSpawn. replacementWorker.on("message", (raw: unknown) => { - const m = raw as { type: string; pid?: number; message?: string; status?: number }; + const m = raw as WorkerToHostMessage; if (m.type === "error" && m.pid === pid) { finalizeProcessWorkerError(pid, replacementWorker!, m.message); } else if (m.type === "exit" && m.pid === pid) { void finalizeProcessWorker(pid, replacementWorker!, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === pid) { + const process = processes.get(pid); + if (process && process.worker === replacementWorker) { + handleVmInterruptTimer(m, pid, process); + } } }); @@ -1027,10 +1109,11 @@ async function handleExec( const message = err instanceof Error ? err.message : String(err); try { - post({ - type: "stderr", + reportHostDiagnostic({ pid, - data: new TextEncoder().encode(`[exec] post-commit transition failed: ${message}\n`), + status: signalExitStatus(SIGSEGV), + source: "exec post-commit transition", + message: `[exec] post-commit transition failed: ${message}`, }); } catch { // A closed host port must not prevent kernel-side reap. @@ -1064,7 +1147,9 @@ async function handleExec( /** * Pre-flight resolver for SYS_SPAWN. Side-effect-free: looks up program * bytes for `path` (via the same execPrograms map + main-thread fallback - * `resolveExec` already uses for execve). Returns null on ENOENT. + * `resolveExec` already uses for execve), follows shebangs, and compiles the + * final Wasm module. Returns null on ENOENT and `{ errno }` when the located + * target cannot be launched. * * `handleSpawn` in `host/src/kernel-worker.ts` calls this BEFORE * `kernel_spawn_process` so that file_actions (which the kernel runs @@ -1079,26 +1164,23 @@ async function handlePosixSpawnResolve( } /** - * Launch a worker for a SYS_SPAWN child whose program bytes have already - * been resolved by `handlePosixSpawnResolve`. The kernel has built the - * child Process descriptor + applied file actions by the time we get - * here, so this just allocates a Memory, registers the process, and - * spawns the worker. + * Launch a worker for a SYS_SPAWN child whose program has already been + * resolved and compiled by `handlePosixSpawnResolve`. The kernel has built + * the child Process descriptor + applied file actions by the time we get + * here, so this just allocates a Memory, registers the process, and spawns + * the worker. */ async function handlePosixSpawn( childPid: number, - programBytes: ArrayBuffer, - argv: string[], + program: ResolvedSpawnProgram, envp: string[], ): Promise { - // The kernel child is authoritative as soon as kernel_spawn_process returns. - // A group-directed signal may already have terminated it before the host - // installs a Worker. Preserve the successful spawn and its waitable zombie, - // but do not resurrect it by registering a new execution generation. + // Preserve a child that became a zombie before launch, but do not resurrect + // it by registering a new execution generation. if (!kernelWorker.shouldLaunchPendingChild(childPid)) return 0; - post({ type: "proc_event", kind: "spawn", pid: childPid }); + const { programBytes, programModule, argv } = program; const ptrWidth = detectPtrWidth(programBytes); const { memory, @@ -1122,6 +1204,7 @@ async function handlePosixSpawn( pid: childPid, ppid: 0, programBytes, + programModule, memory, channelOffset, argv, @@ -1134,6 +1217,7 @@ async function handlePosixSpawn( processes.set(childPid, { memory, programBytes, + programModule, worker: newWorker, channelOffset, ptrWidth, @@ -1144,11 +1228,16 @@ async function handlePosixSpawn( newWorker.on("error", (err: Error) => finalizeUnexpectedWorkerError(childPid, newWorker, "spawn worker error", err)); newWorker.on("message", (raw: unknown) => { - const m = raw as { type: string; pid?: number; message?: string; status?: number }; + const m = raw as WorkerToHostMessage; if (m.type === "error" && m.pid === childPid) { finalizeProcessWorkerError(childPid, newWorker, m.message); } else if (m.type === "exit" && m.pid === childPid) { void finalizeProcessWorker(childPid, newWorker, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === childPid) { + const process = processes.get(childPid); + if (process?.worker === newWorker) { + handleVmInterruptTimer(m, childPid, process); + } } }); @@ -1198,10 +1287,10 @@ async function handleClone( alloc = processInfo.threadAllocator.allocate(memory); } catch (e) { const message = e instanceof Error ? e.message : String(e); - post({ - type: "stderr", + reportHostDiagnostic({ pid, - data: new TextEncoder().encode(`[kernel-worker] pid=${pid}: ${message}\n`), + source: "clone allocation", + message: `[kernel-worker] pid=${pid}: ${message}`, }); throw e; } @@ -1278,9 +1367,15 @@ async function handleClone( void terminateThreadEntry(); return; } - const text = `[kernel-worker] pid=${pid} tid=${tid}: ${reason}\n`; - post({ type: "stderr", pid, data: new TextEncoder().encode(text) }); const disposition = threadWorkerFailureDisposition(reason); + reportHostDiagnostic({ + pid, + status: disposition.kind === "guest-fatal-trap" + ? disposition.exitStatus + : undefined, + source: "thread worker failure", + message: `[kernel-worker] pid=${pid} tid=${tid}: ${reason}`, + }); kernelWorker.finalizeThreadExit(pid, tid, alloc.channelOffset); void terminateThreadEntry(); if (disposition.kind === "guest-fatal-trap") { @@ -1298,6 +1393,10 @@ async function handleClone( void terminateThreadEntry(); } else if (m.type === "error") { failThread(m.message); + } else if (m.type === "vm_interrupt_timer") { + if (isCurrentThreadGeneration() && m.pid === pid) { + handleVmInterruptTimer(m, pid, processInfo); + } } }); threadWorker.on("error", (err: Error) => failThread(`worker error: ${err.message ?? err}`)); @@ -1321,6 +1420,7 @@ async function finishProcessExit( if (!expectedWorker) return; const info = processes.get(pid); if (!info || info.worker !== expectedWorker) return; + vmInterruptTimers.clear(pid); const existingTeardown = processTeardowns.get(expectedWorker); if (existingTeardown) { @@ -1368,6 +1468,7 @@ async function finishProcessExit( async function handleTerminate(msg: TerminateProcessMessage) { const pid = msg.pid; + vmInterruptTimers.clear(pid); // Terminate thread workers const threads = threadWorkers.get(pid); @@ -1404,6 +1505,7 @@ async function handleTerminate(msg: TerminateProcessMessage) { async function handleDestroy(msg: { requestId: number }) { const processEntries = [...processes.entries()]; for (const [pid, info] of processEntries) { + vmInterruptTimers.clear(pid); await terminateThreadWorkers(pid); await terminateTrackedWorker(info.worker); try { kernelWorker.unregisterProcess(pid); } catch {} @@ -1419,8 +1521,10 @@ async function handleDestroy(msg: { requestId: number }) { } } processes.clear(); + vmInterruptTimers.clearAll(); processTeardowns.clear(); reportedExits.clear(); + reportedNonzeroProcessExits.clear(); threadModuleCache.clear(); threadWorkers.clear(); ptyByPid.clear(); @@ -1565,5 +1669,13 @@ port.on("message", (msg: MainToKernelMessage) => { case "kms_attach_stats": kernelWorker.attachKmsStats(msg.crtcId, msg.stats); break; + default: { + const exhaustive: never = msg; + void exhaustive; + reportWorkerProtocolError( + `unknown main-thread message type: ${String((msg as { type?: unknown }).type)}`, + ); + break; + } } }); diff --git a/host/src/platform/native-metadata.ts b/host/src/platform/native-metadata.ts index b90463a3a..ef1954ddd 100644 --- a/host/src/platform/native-metadata.ts +++ b/host/src/platform/native-metadata.ts @@ -11,6 +11,11 @@ interface VirtualMetadata { mode?: number; uid?: number; gid?: number; + atimeMs?: number; + mtimeMs?: number; + nativeAtimeMs?: number; + nativeMtimeMs?: number; + nativeCtimeMs?: number; ctimeMs?: number; } @@ -24,8 +29,14 @@ interface VirtualMetadata { export class NativeMetadataOverlay { private readonly entries = new Map(); + constructor( + private readonly defaultUid = 0, + private readonly defaultGid = 0, + ) {} + toStatResult(s: Stats): StatResult { const metadata = this.entries.get(this.key(s)); + if (metadata !== undefined) this.reconcileNativeTimes(metadata, s); return { dev: s.dev, ino: s.ino, @@ -33,12 +44,14 @@ export class NativeMetadataOverlay { ? s.mode : (s.mode & ~MODE_CHANGE_MASK) | (metadata.mode & MODE_CHANGE_MASK), nlink: s.nlink, - uid: metadata?.uid ?? 0, - gid: metadata?.gid ?? 0, + uid: metadata?.uid ?? this.defaultUid, + gid: metadata?.gid ?? this.defaultGid, size: s.size, - atimeMs: s.atimeMs, - mtimeMs: s.mtimeMs, - ctimeMs: metadata?.ctimeMs ?? s.ctimeMs, + atimeMs: metadata?.atimeMs ?? s.atimeMs, + mtimeMs: metadata?.mtimeMs ?? s.mtimeMs, + ctimeMs: metadata?.ctimeMs === undefined + ? s.ctimeMs + : Math.max(metadata.ctimeMs, s.ctimeMs), }; } @@ -55,6 +68,27 @@ export class NativeMetadataOverlay { metadata.ctimeMs = Date.now(); } + utimens( + s: Stats, + atimeMs: number, + mtimeMs: number, + nativeAfter: Stats, + ): void { + const metadata = this.metadataFor(s); + metadata.atimeMs = atimeMs; + metadata.mtimeMs = mtimeMs; + metadata.nativeAtimeMs = nativeAfter.atimeMs; + metadata.nativeMtimeMs = nativeAfter.mtimeMs; + metadata.nativeCtimeMs = nativeAfter.ctimeMs; + metadata.ctimeMs = Math.max(metadata.ctimeMs ?? 0, nativeAfter.ctimeMs); + } + + noteNativeContentChange(s: Stats): void { + const metadata = this.entries.get(this.key(s)); + if (metadata === undefined) return; + this.clearTimeOverrides(metadata); + } + forget(s: Stats): void { this.entries.delete(this.key(s)); } @@ -76,6 +110,38 @@ export class NativeMetadataOverlay { return metadata; } + private reconcileNativeTimes(metadata: VirtualMetadata, s: Stats): void { + if (metadata.nativeCtimeMs === undefined) return; + + const nativeMetadataChanged = s.ctimeMs !== metadata.nativeCtimeMs; + if ( + nativeMetadataChanged || + (metadata.nativeAtimeMs !== undefined && s.atimeMs !== metadata.nativeAtimeMs) + ) { + delete metadata.atimeMs; + delete metadata.nativeAtimeMs; + } + if ( + nativeMetadataChanged || + (metadata.nativeMtimeMs !== undefined && s.mtimeMs !== metadata.nativeMtimeMs) + ) { + delete metadata.mtimeMs; + delete metadata.nativeMtimeMs; + } + + if (metadata.atimeMs === undefined && metadata.mtimeMs === undefined) { + delete metadata.nativeCtimeMs; + } + } + + private clearTimeOverrides(metadata: VirtualMetadata): void { + delete metadata.atimeMs; + delete metadata.mtimeMs; + delete metadata.nativeAtimeMs; + delete metadata.nativeMtimeMs; + delete metadata.nativeCtimeMs; + } + private key(s: Stats): string { return `${s.dev}:${s.ino}`; } diff --git a/host/src/platform/node.ts b/host/src/platform/node.ts index a5f82b367..8c9c4a273 100644 --- a/host/src/platform/node.ts +++ b/host/src/platform/node.ts @@ -13,6 +13,9 @@ import type { PlatformIO, StatResult, StatfsResult } from "../types"; import { nativeStatfs, translateOpenFlags } from "../vfs/host-fs"; import { NativeMetadataOverlay } from "./native-metadata"; +const UTIME_NOW = 0x3fffffff; +const UTIME_OMIT = 0x3ffffffe; + export class NodePlatformIO implements PlatformIO { private dirHandles = new Map(); private nextDirHandle = 1; @@ -97,6 +100,7 @@ export class NodePlatformIO implements PlatformIO { ): number { const pos = offset ?? this.fdPositions.get(handle) ?? 0; const bytesWritten = fs.writeSync(handle, buffer, 0, length, pos); + if (bytesWritten > 0) this.metadata.noteNativeContentChange(fs.fstatSync(handle)); if (offset === null) { this.fdPositions.set(handle, pos + bytesWritten); } @@ -218,9 +222,24 @@ export class NodePlatformIO implements PlatformIO { } utimensat(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { - const atime = atimeSec + atimeNsec / 1e9; - const mtime = mtimeSec + mtimeNsec / 1e9; - fs.utimesSync(this.rewritePath(path), atime, mtime); + const nativePath = this.rewritePath(path); + if (atimeNsec === UTIME_OMIT && mtimeNsec === UTIME_OMIT) return; + + const stat = fs.statSync(nativePath); + const current = this.metadata.toStatResult(stat); + const nowMs = Date.now(); + const atimeMs = atimeNsec === UTIME_OMIT + ? current.atimeMs + : atimeNsec === UTIME_NOW + ? nowMs + : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); + const mtimeMs = mtimeNsec === UTIME_OMIT + ? current.mtimeMs + : mtimeNsec === UTIME_NOW + ? nowMs + : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); + fs.utimesSync(nativePath, atimeMs / 1000, mtimeMs / 1000); + this.metadata.utimens(stat, atimeMs, mtimeMs, fs.statSync(nativePath)); } opendir(path: string): number { @@ -258,6 +277,7 @@ export class NodePlatformIO implements PlatformIO { ftruncate(handle: number, length: number): void { fs.ftruncateSync(handle, length); + this.metadata.noteNativeContentChange(fs.fstatSync(handle)); } fsync(handle: number): void { @@ -282,8 +302,8 @@ export class NodePlatformIO implements PlatformIO { const elapsed = ns - this._startNs; return { sec: Number(elapsed / 1000000000n), nsec: Number(elapsed % 1000000000n) }; } - if (clockId === 1) { - // CLOCK_MONOTONIC + if (clockId === 1 || clockId === 7) { + // CLOCK_MONOTONIC / CLOCK_BOOTTIME return { sec: Number(ns / 1000000000n), nsec: Number(ns % 1000000000n) }; } // CLOCK_REALTIME — use hrtime + epoch offset for nanosecond resolution diff --git a/host/src/vfs/host-fs.ts b/host/src/vfs/host-fs.ts index 11d57d883..79d12221d 100644 --- a/host/src/vfs/host-fs.ts +++ b/host/src/vfs/host-fs.ts @@ -12,6 +12,9 @@ import { NativeMetadataOverlay } from "../platform/native-metadata"; import type { FileSystemBackend, DirEntry } from "./types"; import { DEFAULT_STATFS_BLOCK_SIZE, DEFAULT_STATFS_NAMELEN } from "../statfs"; +const UTIME_NOW = 0x3fffffff; +const UTIME_OMIT = 0x3ffffffe; + /** * Translate Linux/POSIX open flags (as used by musl libc) to the * platform-native flag values that Node.js `fs.openSync` expects. @@ -42,11 +45,11 @@ export function translateOpenFlags(linuxFlags: number): number { if (linuxFlags & L_O_TRUNC) native |= fs.constants.O_TRUNC; if (linuxFlags & L_O_APPEND) native |= fs.constants.O_APPEND; if (linuxFlags & L_O_NONBLOCK) native |= fs.constants.O_NONBLOCK; - if ((linuxFlags & L_O_DIRECTORY) && fs.constants.O_DIRECTORY) + if (linuxFlags & L_O_DIRECTORY && fs.constants.O_DIRECTORY) native |= fs.constants.O_DIRECTORY; - if ((linuxFlags & L_O_NOFOLLOW) && fs.constants.O_NOFOLLOW) + if (linuxFlags & L_O_NOFOLLOW && fs.constants.O_NOFOLLOW) native |= fs.constants.O_NOFOLLOW; - if ((linuxFlags & L_O_NOCTTY) && fs.constants.O_NOCTTY) + if (linuxFlags & L_O_NOCTTY && fs.constants.O_NOCTTY) native |= fs.constants.O_NOCTTY; // O_LARGEFILE and O_CLOEXEC have no Node.js equivalent; ignored. @@ -83,31 +86,158 @@ export function nativeStatfs(path: string): StatfsResult { export class HostFileSystem implements FileSystemBackend { private rootPath: string; + private guestMountPoint: string; private fdPositions = new Map(); private dirHandles = new Map(); private nextDirHandle = 1; - private metadata = new NativeMetadataOverlay(); - - constructor(rootPath: string) { - this.rootPath = nodePath.resolve(rootPath); + private metadata: NativeMetadataOverlay; + + constructor( + rootPath: string, + guestMountPoint = "/", + options: { uid?: number; gid?: number } = {}, + ) { + const resolvedRoot = nodePath.resolve(rootPath); + this.rootPath = fs.existsSync(resolvedRoot) + ? fs.realpathSync(resolvedRoot) + : resolvedRoot; + this.guestMountPoint = this.normalizeGuestMountPoint(guestMountPoint); + this.metadata = new NativeMetadataOverlay( + options.uid ?? 0, + options.gid ?? 0, + ); } /** - * Resolve a mount-relative path to an absolute host path, - * ensuring it stays within `rootPath`. + * Resolve a mount-relative guest path to an absolute host path, ensuring it + * stays within `rootPath`. + * + * This intentionally resolves components one at a time instead of using + * `path.resolve()`. POSIX pathname resolution must look up an intermediate + * component before a following `..` can step back out of it: + * `existing/missing/../file` fails with ENOENT because `missing` is looked + * up as a directory first. Lexical normalization would incorrectly collapse + * that to `existing/file`. + * + * Resolved prefixes are deliberately not cached. A host-backed tree can be + * changed externally or through another mount of the same directory; using + * a stale prefix after a directory-to-symlink replacement would bypass the + * component checks below. + * + * Native symlink targets are stored as guest strings. When following a + * symlink whose target is absolute and still inside this mount, translate it + * back to a mount-relative path before continuing. This preserves readlink(2) + * output while allowing stat/open/chmod to follow absolute in-guest links. */ - private safePath(relative: string): string { - const resolved = nodePath.resolve( - this.rootPath, - relative.replace(/^\//, ""), - ); + private safePath(relative: string, followFinal = true): string { + const hadTrailingSlash = relative.length > 1 && /\/+$/.test(relative); + let current = this.rootPath; + let pending = this.pathParts(relative); + let symlinkDepth = 0; + + while (pending.length > 0) { + const part = pending.shift()!; + if (part === ".") continue; + if (part === "..") { + if (current === this.rootPath) { + throw new Error("EACCES: path traversal blocked"); + } + current = nodePath.dirname(current); + continue; + } + + const candidate = nodePath.join(current, part); + const isFinal = pending.length === 0; + const shouldFollow = !isFinal || followFinal; + + let lst: fs.Stats | null = null; + try { + lst = fs.lstatSync(candidate); + } catch (err: any) { + if (isFinal && err?.code === "ENOENT") { + current = candidate; + break; + } + throw err; + } + + if (shouldFollow && lst.isSymbolicLink()) { + if (++symlinkDepth > 40) + throw new Error("ELOOP: too many symbolic links"); + const target = fs.readlinkSync(candidate, "utf8"); + if (target.startsWith("/")) { + const mountRelative = this.guestAbsoluteToMountRelative(target); + if (mountRelative === null) { + throw new Error("EACCES: absolute symlink target escapes mount"); + } + current = this.rootPath; + pending = [...this.pathParts(mountRelative), ...pending]; + } else { + pending = [...this.pathParts(target), ...pending]; + } + continue; + } + + if (!isFinal && !lst.isDirectory()) { + throw new Error("ENOTDIR: not a directory"); + } + + if (!isFinal) { + current = fs.realpathSync(candidate); + this.assertWithinRoot(current); + } else { + current = candidate; + } + } + + if ( + hadTrailingSlash && + current !== this.rootPath && + !current.endsWith(nodePath.sep) + ) { + // Keep a final separator for native fs calls. POSIX requires a + // trailing slash to resolve the preceding component as a directory; the + // native call then returns ENOTDIR for regular files while still + // permitting operations such as mkdir("new-dir/"). + current += nodePath.sep; + } + this.assertWithinRoot(current); + return current; + } + + private normalizeGuestMountPoint(mountPoint: string): string { + if (!mountPoint.startsWith("/")) mountPoint = `/${mountPoint}`; + return mountPoint !== "/" && mountPoint.endsWith("/") + ? mountPoint.slice(0, -1) + : mountPoint; + } + + private pathParts(path: string): string[] { + return path + .replace(/^\/+/, "") + .split("/") + .filter((part) => part.length > 0 && part !== "."); + } + + private guestAbsoluteToMountRelative(path: string): string | null { + if (this.guestMountPoint === "/") return path; + if (path === this.guestMountPoint) return "/"; + if (path.startsWith(`${this.guestMountPoint}/`)) { + return path.slice(this.guestMountPoint.length) || "/"; + } + return null; + } + + private assertWithinRoot(path: string): void { + const rel = nodePath.relative(this.rootPath, path); + if (rel === "") return; if ( - resolved !== this.rootPath && - !resolved.startsWith(this.rootPath + nodePath.sep) + rel === ".." || + rel.startsWith(`..${nodePath.sep}`) || + nodePath.isAbsolute(rel) ) { throw new Error("EACCES: path traversal blocked"); } - return resolved; } private toStatResult(s: fs.Stats): StatResult { @@ -124,7 +254,10 @@ export class HostFileSystem implements FileSystemBackend { // ── File handle operations ─────────────────────────────────── open(path: string, flags: number, mode: number): number { - const nativePath = this.safePath(path); + const noFollowFinal = + (flags & 0o400000) !== 0 || + ((flags & 0o100) !== 0 && (flags & 0o200) !== 0); + const nativePath = this.safePath(path, !noFollowFinal); const created = (flags & 0o100) !== 0 && !fs.existsSync(nativePath); const fd = fs.openSync(nativePath, translateOpenFlags(flags), mode); if (created) this.metadata.chmod(fs.fstatSync(fd), mode); @@ -160,6 +293,8 @@ export class HostFileSystem implements FileSystemBackend { ): number { const pos = offset ?? this.fdPositions.get(handle) ?? 0; const bytesWritten = fs.writeSync(handle, buffer, 0, length, pos); + if (bytesWritten > 0) + this.metadata.noteNativeContentChange(fs.fstatSync(handle)); if (offset === null) { this.fdPositions.set(handle, pos + bytesWritten); } @@ -191,6 +326,7 @@ export class HostFileSystem implements FileSystemBackend { ftruncate(handle: number, length: number): void { fs.ftruncateSync(handle, length); + this.metadata.noteNativeContentChange(fs.fstatSync(handle)); } fsync(handle: number): void { @@ -212,7 +348,7 @@ export class HostFileSystem implements FileSystemBackend { } lstat(path: string): StatResult { - return this.toStatResult(fs.lstatSync(this.safePath(path))); + return this.toStatResult(fs.lstatSync(this.safePath(path, false))); } statfs(path: string): StatfsResult { @@ -220,45 +356,52 @@ export class HostFileSystem implements FileSystemBackend { } mkdir(path: string, mode: number): void { - const nativePath = this.safePath(path); + const nativePath = this.safePath(path, false); fs.mkdirSync(nativePath, { mode }); this.metadata.chmod(fs.statSync(nativePath), mode); } rmdir(path: string): void { - const nativePath = this.safePath(path); + const nativePath = this.safePath(path, false); const stat = fs.lstatSync(nativePath); fs.rmdirSync(nativePath); this.metadata.forget(stat); } unlink(path: string): void { - const nativePath = this.safePath(path); + const nativePath = this.safePath(path, false); const stat = fs.lstatSync(nativePath); fs.unlinkSync(nativePath); if (stat.nlink <= 1) this.metadata.forget(stat); } rename(oldPath: string, newPath: string): void { - const nativeNewPath = this.safePath(newPath); + const nativeNewPath = this.safePath(newPath, false); let replaced: fs.Stats | undefined; try { replaced = fs.lstatSync(nativeNewPath); } catch {} - fs.renameSync(this.safePath(oldPath), nativeNewPath); - if (replaced !== undefined && replaced.nlink <= 1) this.metadata.forget(replaced); + fs.renameSync(this.safePath(oldPath, false), nativeNewPath); + if (replaced !== undefined && replaced.nlink <= 1) + this.metadata.forget(replaced); } link(existingPath: string, newPath: string): void { - fs.linkSync(this.safePath(existingPath), this.safePath(newPath)); + // Resolve intermediate components ourselves, but leave the final source + // component to native link(2). POSIX permits link() either to follow a + // final symlink or to link the symlink inode; native hosts differ here. + fs.linkSync( + this.safePath(existingPath, false), + this.safePath(newPath, false), + ); } symlink(target: string, path: string): void { - fs.symlinkSync(target, this.safePath(path)); + fs.symlinkSync(target, this.safePath(path, false)); } readlink(path: string): string { - return fs.readlinkSync(this.safePath(path), "utf8"); + return fs.readlinkSync(this.safePath(path, false), "utf8"); } chmod(path: string, mode: number): void { @@ -273,10 +416,33 @@ export class HostFileSystem implements FileSystemBackend { this.metadata.access(fs.statSync(this.safePath(path)), mode); } - utimensat(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { - const atime = atimeSec + atimeNsec / 1e9; - const mtime = mtimeSec + mtimeNsec / 1e9; - fs.utimesSync(this.safePath(path), atime, mtime); + utimensat( + path: string, + atimeSec: number, + atimeNsec: number, + mtimeSec: number, + mtimeNsec: number, + ): void { + const nativePath = this.safePath(path); + if (atimeNsec === UTIME_OMIT && mtimeNsec === UTIME_OMIT) return; + + const stat = fs.statSync(nativePath); + const current = this.metadata.toStatResult(stat); + const nowMs = Date.now(); + const atimeMs = + atimeNsec === UTIME_OMIT + ? current.atimeMs + : atimeNsec === UTIME_NOW + ? nowMs + : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); + const mtimeMs = + mtimeNsec === UTIME_OMIT + ? current.mtimeMs + : mtimeNsec === UTIME_NOW + ? nowMs + : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); + fs.utimesSync(nativePath, atimeMs / 1000, mtimeMs / 1000); + this.metadata.utimens(stat, atimeMs, mtimeMs, fs.statSync(nativePath)); } // ── Directory iteration ───────────────────────────────────── @@ -295,12 +461,18 @@ export class HostFileSystem implements FileSystemBackend { if (!entry) return null; let dtype = 0; // DT_UNKNOWN - if (entry.isFile()) dtype = 8; // DT_REG - else if (entry.isDirectory()) dtype = 4; // DT_DIR - else if (entry.isSymbolicLink()) dtype = 10; // DT_LNK - else if (entry.isFIFO()) dtype = 1; // DT_FIFO - else if (entry.isSocket()) dtype = 12; // DT_SOCK - else if (entry.isCharacterDevice()) dtype = 2; // DT_CHR + if (entry.isFile()) + dtype = 8; // DT_REG + else if (entry.isDirectory()) + dtype = 4; // DT_DIR + else if (entry.isSymbolicLink()) + dtype = 10; // DT_LNK + else if (entry.isFIFO()) + dtype = 1; // DT_FIFO + else if (entry.isSocket()) + dtype = 12; // DT_SOCK + else if (entry.isCharacterDevice()) + dtype = 2; // DT_CHR else if (entry.isBlockDevice()) dtype = 6; // DT_BLK return { name: entry.name, type: dtype, ino: 0 }; diff --git a/host/src/vfs/memory-fs.ts b/host/src/vfs/memory-fs.ts index edc470bce..aaa5239fe 100644 --- a/host/src/vfs/memory-fs.ts +++ b/host/src/vfs/memory-fs.ts @@ -4,6 +4,8 @@ import { SFFS_SUPER_MAGIC } from "../statfs"; import type { FileSystemBackend, DirEntry } from "./types"; import { SharedFS, + type NamespaceEntryIdentity, + type SharedFsIdentityState, type StatResult as SfsStatResult, } from "./sharedfs-vendor"; import type { ZipEntry } from "./zip"; @@ -11,7 +13,13 @@ import type { ZipEntry } from "./zip"; /** Serializable lazy file entry for transfer between instances. */ export interface LazyFileEntry { ino: number; + /** Inode-slot generation; omitted only by legacy serialized metadata. */ + generation?: number; + /** Inode data-mutation sequence; omitted only by legacy metadata. */ + dataSequence?: number; path: string; + /** All hard-link names for this inode; omitted by legacy metadata. */ + paths?: string[]; url: string; size: number; } @@ -37,9 +45,17 @@ export type LazyDownloadListener = (event: LazyDownloadEvent) => void; /** Per-file metadata for a file inside a lazy archive. */ export interface LazyArchiveFileEntry { ino: number; + /** Inode-slot generation; omitted only by legacy serialized metadata. */ + generation?: number; + /** Inode data-mutation sequence; omitted only by legacy metadata. */ + dataSequence?: number; size: number; isSymlink: boolean; deleted: boolean; + /** True once this inode's archive backing is no longer pending. */ + materialized?: boolean; + /** Original path inside the archive (stable across VFS rename/hard-link). */ + archivePath?: string; } /** @@ -61,9 +77,13 @@ export interface SerializedLazyArchiveEntry { entries: Array<{ vfsPath: string; ino: number; + generation?: number; + dataSequence?: number; size: number; isSymlink: boolean; deleted: boolean; + materialized?: boolean; + archivePath?: string; }>; } @@ -118,7 +138,9 @@ const COPY_CHUNK_BYTES = 1024 * 1024; const MIN_REBASE_INITIAL_BYTES = 16 * 1024 * 1024; const VFS_IMAGE_MAX_METADATA_BYTES = 64 * 1024; -function cloneMetadata(metadata: VfsImageMetadata | null): VfsImageMetadata | null { +function cloneMetadata( + metadata: VfsImageMetadata | null, +): VfsImageMetadata | null { return metadata === null ? null : { ...metadata }; } @@ -127,15 +149,22 @@ function validateMetadata(metadata: VfsImageMetadata): VfsImageMetadata { throw new Error("VFS image metadata must be an object"); } if (metadata.version !== 1) { - throw new Error(`Unsupported VFS image metadata version: ${String(metadata.version)}`); + throw new Error( + `Unsupported VFS image metadata version: ${String(metadata.version)}`, + ); } if ( metadata.kernelAbi !== undefined && (!Number.isInteger(metadata.kernelAbi) || metadata.kernelAbi < 0) ) { - throw new Error(`VFS image metadata kernelAbi must be a non-negative integer`); + throw new Error( + `VFS image metadata kernelAbi must be a non-negative integer`, + ); } - if (metadata.createdBy !== undefined && typeof metadata.createdBy !== "string") { + if ( + metadata.createdBy !== undefined && + typeof metadata.createdBy !== "string" + ) { throw new Error("VFS image metadata createdBy must be a string"); } return { ...metadata }; @@ -196,11 +225,7 @@ function parseImageHeader(input: Uint8Array): ParsedImageHeader { throw new Error("VFS image too small"); } - const view = new DataView( - image.buffer, - image.byteOffset, - image.byteLength, - ); + const view = new DataView(image.buffer, image.byteOffset, image.byteLength); const magic = view.getUint32(0, true); if (magic !== VFS_IMAGE_MAGIC) { throw new Error( @@ -270,12 +295,23 @@ function concatChunks(chunks: Uint8Array[], total: number): Uint8Array { export class MemoryFileSystem implements FileSystemBackend { private fs: SharedFS; private imageMetadata: VfsImageMetadata | null; - /** Lazy files: inode → { path, url, size }. Cleared per-inode after materialization. */ - private lazyFiles = new Map(); + /** Lazy files keyed by inode slot + generation (raw inode numbers are reused). */ + private lazyFiles = new Map< + string, + { + ino: number; + generation: number; + dataSequence: number; + path: string; + paths: Set; + url: string; + size: number; + } + >(); /** Lazy archive groups (bundle of files backed by one zip URL). */ private lazyArchiveGroups: LazyArchiveGroup[] = []; - /** Fast lookup: inode → group it belongs to. Cleared per-group after materialization. */ - private lazyArchiveInodes = new Map(); + /** Fast lookup keyed by inode slot + generation. */ + private lazyArchiveInodes = new Map(); private lazyDownloadListeners = new Set(); private constructor(fs: SharedFS, metadata: VfsImageMetadata | null = null) { @@ -283,12 +319,177 @@ export class MemoryFileSystem implements FileSystemBackend { this.imageMetadata = metadata; } + private static inodeKey(ino: number, generation: number): string { + return `${ino}:${generation}`; + } + + private static canAdoptLegacyLazyStub(st: SfsStatResult): boolean { + // Images from before data-sequence tracking stored regular lazy entries as + // untouched zero-length stubs. Current registration performs one initial + // O_TRUNC, so any later mutation sequence (or concrete bytes) is unsafe to + // associate with metadata that cannot name the content version it saw. + return ( + (st.mode & S_IFMT) === S_IFREG && st.size === 0 && st.dataSequence <= 1 + ); + } + + /** + * Reconcile process-local lazy metadata with authoritative SharedFS names. + * The identity map may come from the same transaction as a filesystem + * snapshot, so callers can serialize matching bytes and lazy paths. + */ + private reconcileLazyIdentityState( + identities: Map, + ): void { + for (const [key, entry] of this.lazyFiles) { + const identity = identities.get(key); + if ( + !identity || + identity.dataSequence !== entry.dataSequence || + identity.paths.length === 0 + ) { + this.lazyFiles.delete(key); + continue; + } + entry.paths = new Set(identity.paths); + if (!entry.paths.has(entry.path)) { + entry.path = identity.paths[0]; + } + } + + this.lazyArchiveInodes.clear(); + for (const group of this.lazyArchiveGroups) { + const pendingByIdentity = new Map(); + for (const entry of group.entries.values()) { + if ( + entry.deleted || + entry.materialized || + entry.generation === undefined + ) + continue; + const key = MemoryFileSystem.inodeKey(entry.ino, entry.generation); + if (!pendingByIdentity.has(key)) pendingByIdentity.set(key, entry); + } + + const reconciled = new Map(); + for (const [key, entry] of pendingByIdentity) { + const identity = identities.get(key); + if (!identity || identity.dataSequence !== (entry.dataSequence ?? 0)) + continue; + for (const path of identity.paths) { + reconciled.set(path, { + ...entry, + ino: identity.ino, + generation: identity.generation, + dataSequence: identity.dataSequence, + deleted: false, + materialized: false, + }); + } + if (identity.paths.length > 0) { + this.lazyArchiveInodes.set(key, group); + } + } + group.entries = reconciled; + group.materialized = reconciled.size === 0; + } + } + + private lazyFileForStat(st: SfsStatResult) { + const key = MemoryFileSystem.inodeKey(st.ino, st.generation); + const entry = this.lazyFiles.get(key); + if (entry && entry.dataSequence !== st.dataSequence) { + this.lazyFiles.delete(key); + return undefined; + } + return entry; + } + + private lazyArchiveForStat(st: SfsStatResult) { + const key = MemoryFileSystem.inodeKey(st.ino, st.generation); + const group = this.lazyArchiveInodes.get(key); + if (!group) return undefined; + const entries = Array.from(group.entries.values()).filter( + (entry) => + entry.ino === st.ino && + entry.generation === st.generation && + !entry.deleted && + !entry.materialized, + ); + if (entries.some((entry) => entry.dataSequence === st.dataSequence)) { + return group; + } + this.lazyArchiveInodes.delete(key); + for (const entry of entries) entry.materialized = true; + return undefined; + } + + /** A successful guest data mutation makes any deferred backing obsolete. */ + private invalidateLazyData(st: SfsStatResult): void { + const key = MemoryFileSystem.inodeKey(st.ino, st.generation); + this.lazyFiles.delete(key); + + const group = this.lazyArchiveInodes.get(key); + if (!group) return; + this.lazyArchiveInodes.delete(key); + for (const entry of group.entries.values()) { + if (entry.ino === st.ino && entry.generation === st.generation) { + // Keep the concrete inode in the image, but prevent a later archive + // fetch from overwriting data the guest supplied through any alias. + entry.materialized = true; + } + } + } + + private rewriteLazyNamespacePaths( + source: NamespaceEntryIdentity, + oldPath: string, + newPath: string, + ): void { + const oldBase = oldPath.length > 1 ? oldPath.replace(/\/+$/, "") : oldPath; + const newBase = newPath.length > 1 ? newPath.replace(/\/+$/, "") : newPath; + const oldPrefix = `${oldBase}/`; + const newPrefix = `${newBase}/`; + const sourceKey = MemoryFileSystem.inodeKey(source.ino, source.generation); + const directory = (source.mode & S_IFMT) === S_IFDIR; + const rewrite = (candidate: string): string => + candidate === oldBase + ? newBase + : directory && candidate.startsWith(oldPrefix) + ? newPrefix + candidate.slice(oldPrefix.length) + : candidate; + + for (const [key, lazy] of this.lazyFiles) { + if (!directory && key !== sourceKey) continue; + lazy.paths = new Set(Array.from(lazy.paths, rewrite)); + lazy.path = rewrite(lazy.path); + } + + for (const group of this.lazyArchiveGroups) { + const rewritten = new Map(); + for (const [candidate, entry] of group.entries) { + const entryKey = + entry.generation === undefined + ? null + : MemoryFileSystem.inodeKey(entry.ino, entry.generation); + rewritten.set( + directory || entryKey === sourceKey ? rewrite(candidate) : candidate, + entry, + ); + } + group.entries = rewritten; + } + } + /** Return the underlying SharedArrayBuffer (for sharing with workers). */ get sharedBuffer(): SharedArrayBuffer { return this.fs.buffer as SharedArrayBuffer; } - static create(sab: SharedArrayBuffer, maxSizeBytes?: number): MemoryFileSystem { + static create( + sab: SharedArrayBuffer, + maxSizeBytes?: number, + ): MemoryFileSystem { return new MemoryFileSystem(SharedFS.mkfs(sab, maxSizeBytes)); } @@ -303,24 +504,43 @@ export class MemoryFileSystem implements FileSystemBackend { */ rebaseToNewFileSystem(maxByteLength: number): MemoryFileSystem { if (!Number.isSafeInteger(maxByteLength) || maxByteLength <= 0) { - throw new Error(`Invalid MemoryFileSystem maxByteLength: ${maxByteLength}`); + throw new Error( + `Invalid MemoryFileSystem maxByteLength: ${maxByteLength}`, + ); } - const initialByteLength = Math.min( - maxByteLength, - Math.max(this.sharedBuffer.byteLength, MIN_REBASE_INITIAL_BYTES), - ); const SharedArrayBufferCtor = SharedArrayBuffer as new ( byteLength: number, options?: { maxByteLength?: number }, ) => SharedArrayBuffer; + + // Copy from one quiescent source image. Exporting lazy paths and then + // walking the live SAB would let a peer rename an entry between those two + // operations, making the logical lazy size disagree with the copied path. + const { bytes: sourceBytes, identities } = this.fs.snapshotState(); + this.reconcileLazyIdentityState(identities); + const lazyEntries = this.serializeLazyEntries(); + const lazyArchiveEntries = this.serializeLazyArchiveEntries(); + const sourceSab = new SharedArrayBufferCtor(sourceBytes.byteLength); + new Uint8Array(sourceSab).set(sourceBytes); + const source = new MemoryFileSystem( + SharedFS.mount(sourceSab, { restoreImage: true }), + this.imageMetadata, + ); + source.importLazyEntries(lazyEntries); + source.importLazyArchiveEntries(lazyArchiveEntries); + + const initialByteLength = Math.min( + maxByteLength, + Math.max(sourceBytes.byteLength, MIN_REBASE_INITIAL_BYTES), + ); const sab = new SharedArrayBufferCtor(initialByteLength, { maxByteLength }); const target = MemoryFileSystem.create(sab, maxByteLength); target.setImageMetadata(this.imageMetadata); - const lazyEntries = this.exportLazyEntries(); - const lazyFilePaths = new Set(lazyEntries.map((entry) => entry.path)); - const lazyArchiveEntries = this.exportLazyArchiveEntries(); + const lazyFilePaths = new Set( + lazyEntries.flatMap((entry) => entry.paths ?? [entry.path]), + ); const lazyArchiveStubPaths = new Set(); for (const group of lazyArchiveEntries) { if (group.materialized) continue; @@ -331,19 +551,40 @@ export class MemoryFileSystem implements FileSystemBackend { } } - this.copyPathToFreshFileSystem("/", target, lazyFilePaths, lazyArchiveStubPaths); - - target.importLazyEntries(lazyEntries.map((entry) => ({ - ...entry, - ino: target.lstat(entry.path).ino, - }))); - target.importLazyArchiveEntries(lazyArchiveEntries.map((group) => ({ - ...group, - entries: group.entries.map((entry) => ({ - ...entry, - ino: entry.deleted ? 0 : target.lstat(entry.vfsPath).ino, + source.copyPathToFreshFileSystem( + "/", + target, + lazyFilePaths, + lazyArchiveStubPaths, + new Map(), + ); + + target.importLazyEntries( + lazyEntries.map((entry) => { + const st = target.fs.lstat(entry.path); + return { + ...entry, + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + }; + }), + ); + target.importLazyArchiveEntries( + lazyArchiveEntries.map((group) => ({ + ...group, + entries: group.entries.map((entry) => { + if (entry.deleted) return { ...entry, ino: 0, generation: undefined }; + const st = target.fs.lstat(entry.vfsPath); + return { + ...entry, + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + }; + }), })), - }))); + ); return target; } @@ -367,20 +608,22 @@ export class MemoryFileSystem implements FileSystemBackend { if (this.lazyDownloadListeners.size === 0) return; const stamped: LazyDownloadEvent = { ...event, t: monotonicNow() }; for (const listener of this.lazyDownloadListeners) { - try { listener(stamped); } catch { /* listener errors must not break VFS I/O */ } + try { + listener(stamped); + } catch { + /* listener errors must not break VFS I/O */ + } } } - private async fetchLazyBytes( - details: { - id: string; - kind: LazyDownloadKind; - url: string; - path?: string; - mountPrefix?: string; - fallbackTotalBytes?: number; - }, - ): Promise { + private async fetchLazyBytes(details: { + id: string; + kind: LazyDownloadKind; + url: string; + path?: string; + mountPrefix?: string; + fallbackTotalBytes?: number; + }): Promise { let loadedBytes = 0; let totalBytes = details.fallbackTotalBytes; const base = { @@ -466,23 +709,38 @@ export class MemoryFileSystem implements FileSystemBackend { /** * Register a lazy file: creates an empty stub in SharedFS and records - * metadata so that read() will fetch content on demand via sync XHR. + * metadata for ensureMaterialized() to fetch asynchronously before a + * synchronous read or exec path consumes the file. * Returns the inode number (useful for forwarding to other instances). */ - registerLazyFile(path: string, url: string, size: number, mode = 0o755): number { + registerLazyFile( + path: string, + url: string, + size: number, + mode = 0o755, + ): number { // Ensure parent directories exist const parts = path.split("/").filter(Boolean); let current = ""; for (let i = 0; i < parts.length - 1; i++) { current += "/" + parts[i]; - try { this.fs.mkdir(current, 0o755); } catch { /* exists */ } + try { + this.fs.mkdir(current, 0o755); + } catch { + /* exists */ + } } - // Create empty stub file - const fd = this.fs.open(path, 0o1101, mode); // O_WRONLY | O_CREAT | O_TRUNC - this.fs.close(fd); - // Get inode - const st = this.fs.stat(path); - this.lazyFiles.set(st.ino, { path, url, size }); + const st = this.fs.createLazyStub(path, mode); + this.invalidateLazyData(st); + this.lazyFiles.set(MemoryFileSystem.inodeKey(st.ino, st.generation), { + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + path, + paths: new Set([path]), + url, + size, + }); return st.ino; } @@ -491,26 +749,105 @@ export class MemoryFileSystem implements FileSystemBackend { * Does not create files — assumes the files already exist in the SharedArrayBuffer. */ importLazyEntries(entries: LazyFileEntry[]): void { + this.importLazyEntriesInternal(entries, false); + } + + private importLazyEntriesInternal( + entries: LazyFileEntry[], + trustedLegacySnapshot: boolean, + ): void { for (const e of entries) { - this.lazyFiles.set(e.ino, { path: e.path, url: e.url, size: e.size }); + const isLegacy = + e.generation === undefined || e.dataSequence === undefined; + if (isLegacy && !trustedLegacySnapshot) { + throw new Error( + "Live lazy-file metadata requires inode generation and data sequence", + ); + } + const validPaths = new Set(); + let identity: SfsStatResult | null = null; + for (const path of new Set([e.path, ...(e.paths ?? [])])) { + let st: SfsStatResult; + try { + st = this.fs.stat(path); + } catch { + continue; + } + if (st.ino !== e.ino) continue; + if (e.generation !== undefined && st.generation !== e.generation) { + continue; + } + if (e.dataSequence === undefined) { + if (!MemoryFileSystem.canAdoptLegacyLazyStub(st)) continue; + } else if (st.dataSequence !== e.dataSequence) continue; + identity ??= st; + validPaths.add(path); + } + if (!identity || validPaths.size === 0) continue; + const primaryPath = validPaths.has(e.path) + ? e.path + : validPaths.values().next().value!; + this.lazyFiles.set( + MemoryFileSystem.inodeKey(identity.ino, identity.generation), + { + ino: identity.ino, + generation: identity.generation, + dataSequence: identity.dataSequence, + path: primaryPath, + paths: validPaths, + url: e.url, + size: e.size, + }, + ); } } - /** Export all pending lazy entries for transfer to another instance. */ - exportLazyEntries(): LazyFileEntry[] { + private serializeLazyEntries(): LazyFileEntry[] { const entries: LazyFileEntry[] = []; - for (const [ino, { path, url, size }] of this.lazyFiles) { - entries.push({ ino, path, url, size }); + for (const { + ino, + generation, + dataSequence, + path, + paths, + url, + size, + } of this.lazyFiles.values()) { + entries.push({ + ino, + generation, + dataSequence, + path, + paths: Array.from(paths), + url, + size, + }); } return entries; } + /** Export all pending lazy entries for transfer to another instance. */ + exportLazyEntries(): LazyFileEntry[] { + this.reconcileLazyIdentityState(this.fs.identityState()); + return this.serializeLazyEntries(); + } + /** Return lazy metadata for `path`, following symlinks through stat(). */ getLazyEntry(path: string): LazyFileEntry | null { try { const st = this.fs.stat(path); - const entry = this.lazyFiles.get(st.ino); - return entry ? { ino: st.ino, path: entry.path, url: entry.url, size: entry.size } : null; + const entry = this.lazyFileForStat(st); + return entry + ? { + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + path: entry.path, + paths: Array.from(entry.paths), + url: entry.url, + size: entry.size, + } + : null; } catch { return null; } @@ -522,11 +859,8 @@ export class MemoryFileSystem implements FileSystemBackend { * them with bundler-produced asset URLs. */ rewriteLazyFileUrls(transform: (url: string, path: string) => string): void { - for (const [ino, entry] of this.lazyFiles) { - this.lazyFiles.set(ino, { - ...entry, - url: transform(entry.url, entry.path), - }); + for (const entry of this.lazyFiles.values()) { + entry.url = transform(entry.url, entry.path); } } @@ -562,55 +896,132 @@ export class MemoryFileSystem implements FileSystemBackend { let current = ""; for (let i = 0; i < parts.length - 1; i++) { current += "/" + parts[i]; - try { this.fs.mkdir(current, 0o755); } catch { /* exists */ } + try { + this.fs.mkdir(current, 0o755); + } catch { + /* exists */ + } } - if (ze.isSymlink && symlinkTargets?.has(ze.fileName)) { + if (ze.isSymlink) { + if (!symlinkTargets?.has(ze.fileName)) { + throw new Error( + `Lazy archive symlink target was not provided: ${ze.fileName}`, + ); + } const target = symlinkTargets.get(ze.fileName)!; this.fs.symlink(target, vfsPath); + const st = this.fs.lstat(vfsPath); + const entry: LazyArchiveFileEntry = { + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + size: ze.uncompressedSize, + isSymlink: true, + deleted: false, + materialized: true, + archivePath: ze.fileName, + }; + group.entries.set(vfsPath, entry); } else { - const fd = this.fs.open(vfsPath, 0o1101, ze.mode); // O_WRONLY | O_CREAT | O_TRUNC - this.fs.close(fd); + const st = this.fs.createLazyStub(vfsPath, ze.mode); + this.invalidateLazyData(st); + const entry: LazyArchiveFileEntry = { + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + size: ze.uncompressedSize, + isSymlink: false, + deleted: false, + materialized: false, + archivePath: ze.fileName, + }; + group.entries.set(vfsPath, entry); + this.lazyArchiveInodes.set( + MemoryFileSystem.inodeKey(st.ino, st.generation), + group, + ); } - - const st = this.fs.lstat(vfsPath); - const entry: LazyArchiveFileEntry = { - ino: st.ino, - size: ze.uncompressedSize, - isSymlink: ze.isSymlink, - deleted: false, - }; - group.entries.set(vfsPath, entry); - this.lazyArchiveInodes.set(st.ino, group); } + group.materialized = Array.from(group.entries.values()).every( + (entry) => entry.deleted || entry.materialized, + ); this.lazyArchiveGroups.push(group); return group; } /** Import lazy archive groups from another instance. Assumes stubs already exist. */ importLazyArchiveEntries(serialized: SerializedLazyArchiveEntry[]): void { + this.importLazyArchiveEntriesInternal(serialized, false); + } + + private importLazyArchiveEntriesInternal( + serialized: SerializedLazyArchiveEntry[], + trustedLegacySnapshot: boolean, + ): void { for (const s of serialized) { const entries = new Map(); + const normalizedPrefix = s.mountPrefix.replace(/\/+$/, ""); for (const e of s.entries) { + let st: SfsStatResult | null = null; + const materialized = + s.materialized || e.materialized === true || e.isSymlink; + if (!e.deleted && !materialized) { + const isLegacy = + e.generation === undefined || e.dataSequence === undefined; + if (isLegacy && !trustedLegacySnapshot) { + throw new Error( + "Live lazy-archive metadata requires inode generation and data sequence", + ); + } + try { + st = this.fs.lstat(e.vfsPath); + } catch { + continue; + } + if (st.ino !== e.ino) continue; + if (e.generation !== undefined && st.generation !== e.generation) { + continue; + } + if (e.dataSequence === undefined) { + if (!MemoryFileSystem.canAdoptLegacyLazyStub(st)) continue; + } else if (st.dataSequence !== e.dataSequence) continue; + } entries.set(e.vfsPath, { ino: e.ino, + generation: st?.generation ?? e.generation, + dataSequence: st?.dataSequence ?? e.dataSequence, size: e.size, isSymlink: e.isSymlink, deleted: e.deleted, + materialized, + archivePath: + e.archivePath ?? e.vfsPath.slice(normalizedPrefix.length + 1), }); } const group: LazyArchiveGroup = { url: s.url, mountPrefix: s.mountPrefix, - materialized: s.materialized, + materialized: + s.materialized || + Array.from(entries.values()).every( + (entry) => entry.deleted || entry.materialized, + ), entries, }; this.lazyArchiveGroups.push(group); if (!group.materialized) { for (const [, entry] of entries) { - if (!entry.deleted) { - this.lazyArchiveInodes.set(entry.ino, group); + if ( + !entry.deleted && + !entry.materialized && + entry.generation !== undefined + ) { + this.lazyArchiveInodes.set( + MemoryFileSystem.inodeKey(entry.ino, entry.generation), + group, + ); } } } @@ -628,20 +1039,35 @@ export class MemoryFileSystem implements FileSystemBackend { } } - /** Export all lazy archive groups for transfer to another instance. */ - exportLazyArchiveEntries(): SerializedLazyArchiveEntry[] { - return this.lazyArchiveGroups.map((group) => ({ - url: group.url, - mountPrefix: group.mountPrefix, - materialized: group.materialized, - entries: Array.from(group.entries, ([vfsPath, entry]) => ({ + private serializeLazyArchiveEntries(): SerializedLazyArchiveEntry[] { + const serialized: SerializedLazyArchiveEntry[] = []; + for (const group of this.lazyArchiveGroups) { + const entries = Array.from(group.entries, ([vfsPath, entry]) => ({ vfsPath, ino: entry.ino, + generation: entry.generation, + dataSequence: entry.dataSequence, size: entry.size, isSymlink: entry.isSymlink, deleted: entry.deleted, - })), - })); + materialized: entry.materialized, + archivePath: entry.archivePath, + })).filter((entry) => !entry.deleted && !entry.materialized); + if (entries.length === 0) continue; + serialized.push({ + url: group.url, + mountPrefix: group.mountPrefix, + materialized: false, + entries, + }); + } + return serialized; + } + + /** Export all pending lazy archive groups for transfer to another instance. */ + exportLazyArchiveEntries(): SerializedLazyArchiveEntry[] { + this.reconcileLazyIdentityState(this.fs.identityState()); + return this.serializeLazyArchiveEntries(); } /** @@ -651,33 +1077,58 @@ export class MemoryFileSystem implements FileSystemBackend { * Returns true if something was materialized, false if already concrete. */ async ensureMaterialized(path: string): Promise { - if (this.lazyFiles.size === 0 && this.lazyArchiveInodes.size === 0) return false; - try { - const st = this.fs.stat(path); // follows symlinks - const entry = this.lazyFiles.get(st.ino); - if (entry) { - const data = await this.fetchLazyBytes({ - id: `file:${st.ino}`, - kind: "file", - url: entry.url, - path: entry.path, - fallbackTotalBytes: entry.size, - }); - const fd = this.fs.open(entry.path, 0o1101, 0o755); // O_WRONLY | O_CREAT | O_TRUNC - this.fs.write(fd, data); - this.fs.close(fd); - this.lazyFiles.delete(st.ino); - return true; - } - const group = this.lazyArchiveInodes.get(st.ino); - if (group) { - await this.ensureArchiveMaterialized(group); - return true; - } + if (this.lazyFiles.size === 0 && this.lazyArchiveInodes.size === 0) return false; + let st: SfsStatResult; + try { + st = this.fs.stat(path); // follows symlinks } catch { return false; } + const key = MemoryFileSystem.inodeKey(st.ino, st.generation); + const entry = this.lazyFiles.get(key); + if (entry) { + const data = await this.fetchLazyBytes({ + id: `file:${st.ino}`, + kind: "file", + url: entry.url, + path: entry.path, + fallbackTotalBytes: entry.size, + }); + for (let attempt = 0; attempt < 3; attempt++) { + if (this.lazyFiles.get(key) !== entry) return false; + for (const candidate of new Set([path, ...entry.paths])) { + const materialized = this.fs.replaceIfIdentity( + candidate, + entry.ino, + entry.generation, + entry.dataSequence, + data, + ); + if (materialized) { + entry.path = candidate; + this.lazyFiles.delete(key); + return true; + } + } + // A peer may have renamed the inode while the fetch was in flight. + // Refresh aliases and retry immediately with the bytes already read. + this.reconcileLazyIdentityState(this.fs.identityState()); + } + throw new Error( + `Lazy file kept changing names while materializing: ${path}`, + ); + } + const group = this.lazyArchiveInodes.get(key); + if (group) { + await this.ensureArchiveMaterialized(group, { + path, + ino: st.ino, + generation: st.generation, + }); + return !this.lazyArchiveInodes.has(key); + } + return false; } /** @@ -685,7 +1136,10 @@ export class MemoryFileSystem implements FileSystemBackend { * central directory, and write every non-deleted entry into its stub. * Subsequent calls are no-ops. */ - async ensureArchiveMaterialized(group: LazyArchiveGroup): Promise { + async ensureArchiveMaterialized( + group: LazyArchiveGroup, + requested?: { path: string; ino: number; generation: number }, + ): Promise { if (group.materialized) return; const zipData = await this.fetchLazyBytes({ @@ -701,21 +1155,98 @@ export class MemoryFileSystem implements FileSystemBackend { for (const ze of zipEntries) zipLookup.set(ze.fileName, ze); const normalizedPrefix = group.mountPrefix.replace(/\/+$/, ""); - for (const [vfsPath, archiveEntry] of group.entries) { - if (archiveEntry.deleted) continue; - if (archiveEntry.isSymlink) continue; // symlinks already created at registration - const zipFileName = vfsPath.slice(normalizedPrefix.length + 1); - const ze = zipLookup.get(zipFileName); - if (!ze) continue; - const content = extractZipEntry(zipData, ze); - const fd = this.fs.open(vfsPath, 0o1101, 0o755); // O_WRONLY | O_CREAT | O_TRUNC - if (content.length > 0) this.fs.write(fd, content); - this.fs.close(fd); + const requestedKey = requested + ? MemoryFileSystem.inodeKey(requested.ino, requested.generation) + : null; + for (let attempt = 0; attempt < 3; attempt++) { + for (const [vfsPath, archiveEntry] of group.entries) { + if (archiveEntry.deleted || archiveEntry.materialized) continue; + if (archiveEntry.isSymlink) { + archiveEntry.materialized = true; + continue; + } + const zipFileName = + archiveEntry.archivePath ?? + vfsPath.slice(normalizedPrefix.length + 1); + const ze = zipLookup.get(zipFileName); + if (!ze) continue; + const content = extractZipEntry(zipData, ze); + if (archiveEntry.generation === undefined) continue; + const key = MemoryFileSystem.inodeKey( + archiveEntry.ino, + archiveEntry.generation, + ); + if (this.lazyArchiveInodes.get(key) !== group) continue; + const candidates = new Set([vfsPath]); + if ( + requested && + requested.ino === archiveEntry.ino && + requested.generation === archiveEntry.generation + ) + candidates.add(requested.path); + let materialized = false; + for (const candidate of candidates) { + materialized = this.fs.replaceIfIdentity( + candidate, + archiveEntry.ino, + archiveEntry.generation, + archiveEntry.dataSequence ?? 0, + content, + ); + if (materialized) break; + } + if (!materialized) continue; + this.lazyArchiveInodes.delete(key); + for (const alias of group.entries.values()) { + if ( + alias.ino === archiveEntry.ino && + alias.generation === archiveEntry.generation + ) + alias.materialized = true; + } + } + + group.materialized = Array.from(group.entries.values()).every( + (entry) => entry.deleted || entry.materialized, + ); + if (group.materialized) return; + this.reconcileLazyIdentityState(this.fs.identityState()); + if (requestedKey && !this.lazyArchiveInodes.has(requestedKey)) return; } - group.materialized = true; - for (const [, archiveEntry] of group.entries) { - this.lazyArchiveInodes.delete(archiveEntry.ino); + if (requestedKey && this.lazyArchiveInodes.has(requestedKey)) { + throw new Error( + `Lazy archive member kept changing names while materializing: ${requested?.path}`, + ); + } + } + + private async materializeAllLazyEntries(): Promise { + // A peer can rename an inode while an asynchronous fetch is in flight. + // Refresh and retry a bounded number of times; a continuously mutating + // filesystem is not a stable source for a self-contained image. + for (let attempt = 0; attempt < 3; attempt++) { + this.reconcileLazyIdentityState(this.fs.identityState()); + if (this.lazyFiles.size === 0 && this.lazyArchiveInodes.size === 0) + return; + + const filePaths = Array.from( + this.lazyFiles.values(), + (entry) => entry.path, + ); + for (const path of filePaths) await this.ensureMaterialized(path); + + const archiveGroups = new Set(this.lazyArchiveInodes.values()); + for (const group of archiveGroups) { + await this.ensureArchiveMaterialized(group); + } + } + + this.reconcileLazyIdentityState(this.fs.identityState()); + if (this.lazyFiles.size !== 0 || this.lazyArchiveInodes.size !== 0) { + throw new Error( + "Cannot create a self-contained VFS image while lazy entries remain pending", + ); } } @@ -729,28 +1260,25 @@ export class MemoryFileSystem implements FileSystemBackend { */ async saveImage(options?: VfsImageOptions): Promise { if (options?.materializeAll) { - const paths = Array.from(this.lazyFiles.values()).map((e) => e.path); - for (const p of paths) { - await this.ensureMaterialized(p); - } + await this.materializeAllLazyEntries(); } - const sabBytes = new Uint8Array(this.fs.buffer); - const lazyEntries = this.exportLazyEntries(); + const { bytes: sabBytes, identities } = this.fs.snapshotState(); + this.reconcileLazyIdentityState(identities); + const lazyEntries = this.serializeLazyEntries(); const hasLazy = lazyEntries.length > 0; const lazyJson = hasLazy ? new TextEncoder().encode(JSON.stringify(lazyEntries)) : new Uint8Array(0); - const archiveEntries = this.exportLazyArchiveEntries(); + const archiveEntries = this.serializeLazyArchiveEntries(); const hasArchives = archiveEntries.length > 0; const archiveJson = hasArchives ? new TextEncoder().encode(JSON.stringify(archiveEntries)) : new Uint8Array(0); - const metadata = options?.metadata === undefined - ? this.imageMetadata - : options.metadata; + const metadata = + options?.metadata === undefined ? this.imageMetadata : options.metadata; const metadataJson = encodeMetadata(metadata); const hasMetadata = metadataJson.byteLength > 0; @@ -760,11 +1288,11 @@ export class MemoryFileSystem implements FileSystemBackend { const metadataSectionSize = hasMetadata ? 4 + metadataJson.byteLength : 0; const totalSize = VFS_IMAGE_HEADER_SIZE + - sabBytes.byteLength + - 4 + - lazyJson.byteLength + - archiveSectionSize + - metadataSectionSize; + sabBytes.byteLength + + 4 + + lazyJson.byteLength + + archiveSectionSize + + metadataSectionSize; const image = new Uint8Array(totalSize); const view = new DataView(image.buffer); @@ -780,10 +1308,8 @@ export class MemoryFileSystem implements FileSystemBackend { ); view.setUint32(12, sabBytes.byteLength, true); - // SAB data — copy from SharedArrayBuffer (can't use set() directly on SAB-backed views in all environments) - const sabCopy = new Uint8Array(sabBytes.byteLength); - sabCopy.set(sabBytes); - image.set(sabCopy, VFS_IMAGE_HEADER_SIZE); + // SAB data is already a detached, runtime-state-free snapshot. + image.set(sabBytes, VFS_IMAGE_HEADER_SIZE); // Lazy entries const lazyOffset = VFS_IMAGE_HEADER_SIZE + sabBytes.byteLength; @@ -801,7 +1327,8 @@ export class MemoryFileSystem implements FileSystemBackend { // Metadata if (hasMetadata) { - const metadataOffset = lazyOffset + 4 + lazyJson.byteLength + archiveSectionSize; + const metadataOffset = + lazyOffset + 4 + lazyJson.byteLength + archiveSectionSize; view.setUint32(metadataOffset, metadataJson.byteLength, true); image.set(metadataJson, metadataOffset + 4); } @@ -833,7 +1360,10 @@ export class MemoryFileSystem implements FileSystemBackend { } if (metadataLen === 0) return null; return decodeMetadata( - parsed.image.subarray(metadataOffset + 4, metadataOffset + 4 + metadataLen), + parsed.image.subarray( + metadataOffset + 4, + metadataOffset + 4 + metadataLen, + ), ); } @@ -866,7 +1396,10 @@ export class MemoryFileSystem implements FileSystemBackend { * so the filesystem can expand beyond the image's original size, up to the * maximum already recorded in the image superblock. */ - static fromImage(image: Uint8Array, options?: { maxByteLength?: number }): MemoryFileSystem { + static fromImage( + image: Uint8Array, + options?: { maxByteLength?: number }, + ): MemoryFileSystem { const parsed = parseImageHeader(image); image = parsed.image; const view = parsed.view; @@ -885,25 +1418,33 @@ export class MemoryFileSystem implements FileSystemBackend { ) => SharedArrayBuffer; const sab = new SharedArrayBufferCtor(sabLen, sabOptions); const sabView = new Uint8Array(sab); - sabView.set(image.subarray(VFS_IMAGE_HEADER_SIZE, VFS_IMAGE_HEADER_SIZE + sabLen)); + sabView.set( + image.subarray(VFS_IMAGE_HEADER_SIZE, VFS_IMAGE_HEADER_SIZE + sabLen), + ); let metadata: VfsImageMetadata | null = null; if (flags & VFS_IMAGE_FLAG_HAS_METADATA) { metadata = MemoryFileSystem.readImageMetadata(image); } - const mfs = new MemoryFileSystem(SharedFS.mount(sab), metadata); + const mfs = new MemoryFileSystem( + SharedFS.mount(sab, { restoreImage: true }), + metadata, + ); // Restore lazy entries const lazyOffset = VFS_IMAGE_HEADER_SIZE + sabLen; const lazyLen = view.getUint32(lazyOffset, true); if (flags & VFS_IMAGE_FLAG_HAS_LAZY) { if (lazyLen > 0) { - const lazyBytes = image.subarray(lazyOffset + 4, lazyOffset + 4 + lazyLen); + const lazyBytes = image.subarray( + lazyOffset + 4, + lazyOffset + 4 + lazyLen, + ); const entries: LazyFileEntry[] = JSON.parse( new TextDecoder().decode(lazyBytes), ); - mfs.importLazyEntries(entries); + mfs.importLazyEntriesInternal(entries, true); } } @@ -922,7 +1463,7 @@ export class MemoryFileSystem implements FileSystemBackend { const entries: SerializedLazyArchiveEntry[] = JSON.parse( new TextDecoder().decode(archiveBytes), ); - mfs.importLazyArchiveEntries(entries); + mfs.importLazyArchiveEntriesInternal(entries, true); } } @@ -944,8 +1485,37 @@ export class MemoryFileSystem implements FileSystemBackend { }; } + private adaptStatWithLazySize(s: SfsStatResult): StatResult { + const result = this.adaptStat(s); + const entry = this.lazyFileForStat(s); + if (entry) { + result.size = entry.size; + return result; + } + + const group = this.lazyArchiveForStat(s); + if (group) { + for (const archiveEntry of group.entries.values()) { + if ( + archiveEntry.ino === s.ino && + archiveEntry.generation === s.generation && + !archiveEntry.deleted + ) { + result.size = archiveEntry.size; + break; + } + } + } + return result; + } + open(path: string, flags: number, mode: number): number { - return this.fs.open(path, flags, mode); + const handle = this.fs.open(path, flags, mode); + if ((flags & 0x0200) !== 0) { + // O_TRUNC + this.invalidateLazyData(this.fs.fstat(handle)); + } + return handle; } close(handle: number): number { @@ -982,9 +1552,12 @@ export class MemoryFileSystem implements FileSystemBackend { this.fs.lseek(handle, offset, 0); // SEEK_SET const n = this.fs.write(handle, buffer.subarray(0, length)); this.fs.lseek(handle, savedPos, 0); // restore position + if (n > 0) this.invalidateLazyData(this.fs.fstat(handle)); return n; } - return this.fs.write(handle, buffer.subarray(0, length)); + const n = this.fs.write(handle, buffer.subarray(0, length)); + if (n > 0) this.invalidateLazyData(this.fs.fstat(handle)); + return n; } seek(handle: number, offset: number, whence: number): number { @@ -992,27 +1565,12 @@ export class MemoryFileSystem implements FileSystemBackend { } fstat(handle: number): StatResult { - const result = this.adaptStat(this.fs.fstat(handle)); - // Override size for unmaterialized lazy files / archive entries - const entry = this.lazyFiles.get(result.ino); - if (entry) { - result.size = entry.size; - } else { - const group = this.lazyArchiveInodes.get(result.ino); - if (group) { - for (const archiveEntry of group.entries.values()) { - if (archiveEntry.ino === result.ino) { - result.size = archiveEntry.size; - break; - } - } - } - } - return result; + return this.adaptStatWithLazySize(this.fs.fstat(handle)); } ftruncate(handle: number, length: number): void { this.fs.ftruncate(handle, length); + this.invalidateLazyData(this.fs.fstat(handle)); } // SharedFS is memory-backed, fsync is a no-op @@ -1026,43 +1584,11 @@ export class MemoryFileSystem implements FileSystemBackend { } stat(path: string): StatResult { - const result = this.adaptStat(this.fs.stat(path)); - // Override size for unmaterialized lazy files / archive entries - const entry = this.lazyFiles.get(result.ino); - if (entry) { - result.size = entry.size; - } else { - const group = this.lazyArchiveInodes.get(result.ino); - if (group) { - for (const archiveEntry of group.entries.values()) { - if (archiveEntry.ino === result.ino) { - result.size = archiveEntry.size; - break; - } - } - } - } - return result; + return this.adaptStatWithLazySize(this.fs.stat(path)); } lstat(path: string): StatResult { - const result = this.adaptStat(this.fs.lstat(path)); - // Override size for unmaterialized lazy files / archive entries - const entry = this.lazyFiles.get(result.ino); - if (entry) { - result.size = entry.size; - } else { - const group = this.lazyArchiveInodes.get(result.ino); - if (group) { - for (const archiveEntry of group.entries.values()) { - if (archiveEntry.ino === result.ino) { - result.size = archiveEntry.size; - break; - } - } - } - } - return result; + return this.adaptStatWithLazySize(this.fs.lstat(path)); } statfs(path: string): StatfsResult { @@ -1092,28 +1618,119 @@ export class MemoryFileSystem implements FileSystemBackend { } unlink(path: string): void { - // If the path belongs to an unmaterialized archive group, mark the entry - // as deleted so materialization skips it. - if (this.lazyArchiveInodes.size > 0) { - try { - const st = this.fs.lstat(path); - const group = this.lazyArchiveInodes.get(st.ino); - if (group) { - const entry = group.entries.get(path); - if (entry) entry.deleted = true; - this.lazyArchiveInodes.delete(st.ino); + const removed = this.fs.unlink(path); + const key = MemoryFileSystem.inodeKey(removed.ino, removed.generation); + if ( + removed.linkCount > 1 && + (this.lazyFiles.has(key) || this.lazyArchiveInodes.has(key)) + ) { + // A peer may have added hard-link names this instance never observed. + // Rebuild aliases from SharedFS instead of treating an empty local path + // set as proof that the inode disappeared. + this.reconcileLazyIdentityState(this.fs.identityState()); + return; + } + + const lazy = this.lazyFiles.get(key); + if (lazy) { + lazy.paths.delete(path); + if (removed.linkCount <= 1) { + this.lazyFiles.delete(key); + } else if (lazy.path === path) { + lazy.path = lazy.paths.values().next().value!; + } + } + + const group = this.lazyArchiveInodes.get(key); + if (group) { + const entry = group.entries.get(path); + if (removed.linkCount <= 1) { + for (const candidate of group.entries.values()) { + if ( + candidate.ino === removed.ino && + candidate.generation === removed.generation + ) + candidate.deleted = true; } - } catch { /* not present — unlink will raise the real error */ } + this.lazyArchiveInodes.delete(key); + } else if (entry) { + group.entries.delete(path); + } } - this.fs.unlink(path); } rename(oldPath: string, newPath: string): void { - this.fs.rename(oldPath, newPath); + const { source, replaced } = this.fs.rename(oldPath, newPath); + + if ( + replaced && + replaced.ino === source.ino && + replaced.generation === source.generation + ) + return; + + let reconciledNamespace = false; + if (replaced) { + const replacedKey = MemoryFileSystem.inodeKey( + replaced.ino, + replaced.generation, + ); + if ( + replaced.linkCount > 1 && + (this.lazyFiles.has(replacedKey) || + this.lazyArchiveInodes.has(replacedKey)) + ) { + // The replaced inode survived through a hard link that may have been + // created by a peer. One authoritative reconciliation updates both + // that alias and the source paths changed by rename(). + this.reconcileLazyIdentityState(this.fs.identityState()); + reconciledNamespace = true; + } + + const replacedLazy = this.lazyFiles.get(replacedKey); + if (!reconciledNamespace && replacedLazy) { + replacedLazy.paths.delete(newPath); + if (replaced.linkCount <= 1) { + this.lazyFiles.delete(replacedKey); + } else if (replacedLazy.path === newPath) { + replacedLazy.path = replacedLazy.paths.values().next().value!; + } + } + const replacedGroup = this.lazyArchiveInodes.get(replacedKey); + if (!reconciledNamespace && replacedGroup) { + const entry = replacedGroup.entries.get(newPath); + if (replaced.linkCount <= 1) { + if (entry) entry.deleted = true; + this.lazyArchiveInodes.delete(replacedKey); + } else if (entry) { + replacedGroup.entries.delete(newPath); + } + } + } + + if (!reconciledNamespace) { + this.rewriteLazyNamespacePaths(source, oldPath, newPath); + } } link(existingPath: string, newPath: string): void { - this.fs.link(existingPath, newPath); + const sourceIdentity = this.fs.link(existingPath, newPath); + const key = MemoryFileSystem.inodeKey( + sourceIdentity.ino, + sourceIdentity.generation, + ); + const lazy = this.lazyFiles.get(key); + if (lazy) lazy.paths.add(newPath); + + const group = this.lazyArchiveInodes.get(key); + if (group) { + const source = Array.from(group.entries.values()).find( + (entry) => + entry.ino === sourceIdentity.ino && + entry.generation === sourceIdentity.generation, + ); + if (source) group.entries.set(newPath, { ...source }); + } } symlink(target: string, path: string): void { @@ -1154,7 +1771,12 @@ export class MemoryFileSystem implements FileSystemBackend { this.chmod(path, mode); } - symlinkWithOwner(target: string, path: string, uid: number, gid: number): void { + symlinkWithOwner( + target: string, + path: string, + uid: number, + gid: number, + ): void { this.symlink(target, path); this.lchown(path, uid, gid); } @@ -1164,6 +1786,7 @@ export class MemoryFileSystem implements FileSystemBackend { target: MemoryFileSystem, lazyFilePaths: Set, lazyArchiveStubPaths: Set, + hardLinks: Map, ): void { const st = this.lstat(path); const kind = st.mode & S_IFMT; @@ -1188,6 +1811,7 @@ export class MemoryFileSystem implements FileSystemBackend { target, lazyFilePaths, lazyArchiveStubPaths, + hardLinks, ); } } finally { @@ -1197,8 +1821,16 @@ export class MemoryFileSystem implements FileSystemBackend { return; } + const identity = st.nlink > 1 ? `${st.dev}:${st.ino}` : null; + const existingHardLink = identity ? hardLinks.get(identity) : undefined; + if (existingHardLink) { + target.link(existingHardLink, path); + return; + } + if (kind === S_IFLNK) { target.symlinkWithOwner(this.readlink(path), path, st.uid, st.gid); + if (identity) hardLinks.set(identity, path); return; } @@ -1206,14 +1838,17 @@ export class MemoryFileSystem implements FileSystemBackend { throw new Error(`Unsupported file type while rebasing VFS: ${path}`); } - const isLazyStub = lazyFilePaths.has(path) || lazyArchiveStubPaths.has(path); + const isLazyStub = + lazyFilePaths.has(path) || lazyArchiveStubPaths.has(path); if (isLazyStub) { target.createFileWithOwner(path, mode, st.uid, st.gid, new Uint8Array(0)); MemoryFileSystem.applyTimes(target, path, st); + if (identity) hardLinks.set(identity, path); return; } this.copyRegularFileToFreshFileSystem(path, target, st, mode); + if (identity) hardLinks.set(identity, path); } private copyRegularFileToFreshFileSystem( @@ -1226,7 +1861,9 @@ export class MemoryFileSystem implements FileSystemBackend { let outFd: number | null = null; try { outFd = target.open(path, O_WRONLY_CREAT_TRUNC, mode); - const chunk = new Uint8Array(Math.min(COPY_CHUNK_BYTES, Math.max(1, st.size))); + const chunk = new Uint8Array( + Math.min(COPY_CHUNK_BYTES, Math.max(1, st.size)), + ); let remaining = st.size; while (remaining > 0) { const wanted = Math.min(chunk.byteLength, remaining); @@ -1258,7 +1895,11 @@ export class MemoryFileSystem implements FileSystemBackend { MemoryFileSystem.applyTimes(target, path, st); } - private static applyTimes(fs: MemoryFileSystem, path: string, st: StatResult): void { + private static applyTimes( + fs: MemoryFileSystem, + path: string, + st: StatResult, + ): void { const atimeSec = Math.floor(st.atimeMs / 1000); const atimeNsec = Math.floor((st.atimeMs - atimeSec * 1000) * 1_000_000); const mtimeSec = Math.floor(st.mtimeMs / 1000); @@ -1271,7 +1912,13 @@ export class MemoryFileSystem implements FileSystemBackend { this.fs.stat(path); } - utimensat(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { + utimensat( + path: string, + atimeSec: number, + atimeNsec: number, + mtimeSec: number, + mtimeNsec: number, + ): void { this.fs.utimens(path, atimeSec, atimeNsec, mtimeSec, mtimeNsec); } @@ -1285,8 +1932,10 @@ export class MemoryFileSystem implements FileSystemBackend { // Determine d_type from mode const mode = entry.stat.mode; let dtype = 0; // DT_UNKNOWN - if ((mode & 0xf000) === 0x8000) dtype = 8; // DT_REG - else if ((mode & 0xf000) === 0x4000) dtype = 4; // DT_DIR + if ((mode & 0xf000) === 0x8000) + dtype = 8; // DT_REG + else if ((mode & 0xf000) === 0x4000) + dtype = 4; // DT_DIR else if ((mode & 0xf000) === 0xa000) dtype = 10; // DT_LNK return { name: entry.name, type: dtype, ino: entry.stat.ino }; } diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index a8488ed88..47ed03713 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -47,6 +47,7 @@ export const O_RDONLY = 0x0000; export const O_WRONLY = 0x0001; export const O_RDWR = 0x0002; export const O_CREAT = 0x0040; +export const O_EXCL = 0x0080; export const O_TRUNC = 0x0200; export const O_APPEND = 0x0400; export const O_DIRECTORY = 0x010000; @@ -65,11 +66,13 @@ export const EPERM = -1; export const ENOENT = -2; export const EIO = -5; export const EBADF = -9; +export const EBUSY = -16; export const EEXIST = -17; export const ENOTDIR = -20; export const EISDIR = -21; export const EINVAL = -22; export const EMFILE = -24; +export const EFBIG = -27; export const ENOSPC = -28; export const ENAMETOOLONG = -36; export const ENOTEMPTY = -39; @@ -92,6 +95,7 @@ const SB_BLOCK_BITMAP_BLOCKS = 48; const SB_INODE_TABLE_BLOCKS = 52; const SB_GENERATION = 56; const SB_GLOBAL_LOCK = 60; +const SB_NAMESPACE_LOCK = 64; const SB_MAX_SIZE_BLOCKS = 68; const SB_GROW_CHUNK_BLOCKS = 72; @@ -108,7 +112,11 @@ const INO_INDIRECT = 88; const INO_DOUBLE_INDIRECT = 92; const INO_UID = 96; // u32 const INO_GID = 100; // u32 -// 104-127 reserved for future fields (flags, xattrs, etc.) +const INO_GENERATION = 104; // uint64, incremented when an inode slot is allocated +const INO_OPEN_COUNT = 112; // u32, open fd references +const INO_DIR_SEQUENCE = 116; // u32, incremented after every directory mutation +const INO_DATA_SEQUENCE = 120; // u32, incremented after explicit data mutation +// 124-127 reserved for future fields (flags, xattrs, etc.) // FD entry layout const FD_INO = 4; @@ -119,11 +127,16 @@ const FD_IS_DIR = 20; // Lock bits const WRITER_BIT = 0x80000000 | 0; // -2147483648 as int32 const READER_MASK = 0x7fffffff | 0; +const MAX_FILE_BLOCKS = + DIRECT_BLOCKS + PTRS_PER_BLOCK + PTRS_PER_BLOCK * PTRS_PER_BLOCK; +const MAX_FILE_SIZE = MAX_FILE_BLOCKS * BLOCK_SIZE; // ── Types ──────────────────────────────────────────────────────────── export interface StatResult { ino: number; + generation: number; + dataSequence: number; mode: number; linkCount: number; size: number; @@ -143,15 +156,51 @@ export interface SharedFsStats { maxName: number; } +export interface SharedFsIdentityState { + ino: number; + generation: number; + dataSequence: number; + paths: string[]; +} + +export interface NamespaceEntryIdentity { + ino: number; + generation: number; + linkCount: number; + mode: number; +} + +export interface RenameIdentityResult { + source: NamespaceEntryIdentity; + replaced?: NamespaceEntryIdentity; +} + +interface DirIndexEntry { + ino: number; + abs: number; + recLen: number; + nameLen: number; +} + +interface DirIndex { + generation: number; + mutationSequence: number; + size: number; + entries: Map; + free: Array<{ abs: number; recLen: number }>; +} + const ERROR_MESSAGES: Record = { [ENOENT]: "No such file or directory", [EIO]: "I/O error", [EBADF]: "Bad file descriptor", + [EBUSY]: "Device or resource busy", [EEXIST]: "File exists", [ENOTDIR]: "Not a directory", [EISDIR]: "Is a directory", [EINVAL]: "Invalid argument", [EMFILE]: "Too many open files", + [EFBIG]: "File too large", [ENOSPC]: "No space left on device", [ENAMETOOLONG]: "File name too long", [ENOTEMPTY]: "Directory not empty", @@ -172,6 +221,11 @@ export class SFSError extends Error { const encoder = new TextEncoder(); const decoder = new TextDecoder(); +const DOTDOT_BYTES = encoder.encode(".."); + +function isReservedDirectoryName(name: string): boolean { + return name === "." || name === ".."; +} /** * Safely decode a Uint8Array that may be backed by SharedArrayBuffer. @@ -194,6 +248,19 @@ export class SharedFS { private view: DataView; private i32: Int32Array; private u8: Uint8Array; + private dirIndexes = new Map(); + private blockAllocHint = 0; + private inodeAllocHint = 2; + private atomicsWaitAllowed: boolean | undefined; + + /** + * Directory operations are stored in ext2-style variable-length entries. + * Workloads such as PHP's bug36365 test create tens of thousands of files in + * one directory. Once a directory reaches this threshold, retain validated + * entry locations so each repeated exact-name lookup does not rescan every + * preceding variable-length record. + */ + private static readonly DIR_INDEX_MIN_SIZE = 64 * 1024; private constructor(public readonly buffer: SharedArrayBuffer) { this.view = new DataView(buffer); @@ -215,14 +282,11 @@ export class SharedFS { // Size inodes for max capacity so we don't run out after growth let totalInodes = Math.floor(maxBlocks / 4); if (totalInodes < 32) totalInodes = 32; - totalInodes = - Math.ceil(totalInodes / INODES_PER_BLOCK) * INODES_PER_BLOCK; + totalInodes = Math.ceil(totalInodes / INODES_PER_BLOCK) * INODES_PER_BLOCK; const inodeBitmapBlocks = Math.ceil(totalInodes / (BLOCK_SIZE * 8)); const blockBitmapBlocks = Math.ceil(maxBlocks / (BLOCK_SIZE * 8)); - const inodeTableBlocks = Math.ceil( - (totalInodes * INODE_SIZE) / BLOCK_SIZE, - ); + const inodeTableBlocks = Math.ceil((totalInodes * INODE_SIZE) / BLOCK_SIZE); const inodeBitmapStart = 1; const blockBitmapStart = inodeBitmapStart + inodeBitmapBlocks; @@ -277,16 +341,19 @@ export class SharedFS { const freeDataBlocks = totalBlocks - dataStart; Atomics.store(fs.i32, SB_FREE_BLOCKS >> 2, freeDataBlocks); + fs.blockAllocHint = dataStart; // Mark inodes 0 and 1 as used const ibStart = inodeBitmapStart * BLOCK_SIZE; fs.i32[ibStart >> 2] |= 0x3; Atomics.store(fs.i32, SB_FREE_INODES >> 2, totalInodes - 2); + fs.inodeAllocHint = 2; // Initialize root inode (inode 1) as empty directory const rootOff = fs.inodeOffset(ROOT_INO); fs.w32(rootOff + INO_MODE, S_IFDIR | 0o755); fs.w32(rootOff + INO_LINK_COUNT, 2); + fs.w64(rootOff + INO_GENERATION, 1); // Allocate a data block for root's directory entries const rootBlock = fs.blockAlloc(); @@ -319,16 +386,165 @@ export class SharedFS { return fs; } - static mount(buffer: SharedArrayBuffer): SharedFS { + static mount( + buffer: SharedArrayBuffer, + options?: { restoreImage?: boolean }, + ): SharedFS { const fs = new SharedFS(buffer); if (fs.r32(SB_MAGIC) !== MAGIC) throw new SFSError(EINVAL, "Bad magic"); if (fs.r32(SB_VERSION) !== VERSION) throw new SFSError(EINVAL, "Bad version"); if (fs.r32(SB_BLOCK_SIZE) !== BLOCK_SIZE) throw new SFSError(EINVAL, "Bad block size"); + if (options?.restoreImage) fs.resetRestoredRuntimeState(); + fs.resetAllocationHints(); return fs; } + /** + * Return a portable, quiescent copy of the filesystem bytes. + * + * File descriptors and inode locks are process-runtime state, not VFS image + * state. Refuse to snapshot while descriptors are live, then clear all lock + * words in the copy so a restored image cannot inherit a dead worker's lock. + */ + snapshotBytes(): Uint8Array { + return this.withNamespaceLock(() => this.snapshotBytesUnlocked()); + } + + snapshotState(): { + bytes: Uint8Array; + identities: Map; + } { + return this.withNamespaceLock(() => { + // Validate quiescence and copy first. With the namespace lock held, no + // new descriptor can appear while the matching path identities are + // collected for lazy metadata. + const bytes = this.snapshotBytesUnlocked(); + return { bytes, identities: this.collectIdentityStateUnlocked() }; + }); + } + + identityState(): Map { + return this.withNamespaceLock(() => this.collectIdentityStateUnlocked()); + } + + private snapshotBytesUnlocked(): Uint8Array { + for (let fd = 0; fd < MAX_FDS; fd++) { + const base = FD_TABLE_OFFSET + fd * FD_ENTRY_SIZE; + if (Atomics.load(this.i32, base >> 2) !== 0) { + throw new SFSError( + EBUSY, + "Cannot save a VFS image with open descriptors", + ); + } + } + + const totalInodes = this.r32(SB_TOTAL_INODES); + for (let ino = 0; ino < totalInodes; ino++) { + const off = this.inodeOffset(ino); + if (this.r32(off + INO_OPEN_COUNT) !== 0) { + throw new SFSError( + EBUSY, + "Cannot save a VFS image with open inode references", + ); + } + } + + const copy = new Uint8Array(this.buffer.byteLength); + copy.set(this.u8); + const view = new DataView(copy.buffer); + view.setUint32(SB_GLOBAL_LOCK, 0, true); + view.setUint32(SB_NAMESPACE_LOCK, 0, true); + copy.fill(0, FD_TABLE_OFFSET, BLOCK_SIZE); + + for (let ino = 0; ino < totalInodes; ino++) { + const off = this.inodeOffset(ino); + view.setUint32(off + INO_LOCK_STATE, 0, true); + view.setUint32(off + INO_OPEN_COUNT, 0, true); + } + return copy; + } + + private collectIdentityStateUnlocked(): Map { + const identities = new Map(); + const directories: Array<{ ino: number; path: string }> = [ + { ino: ROOT_INO, path: "/" }, + ]; + const visitedDirectories = new Set(); + + while (directories.length > 0) { + const directory = directories.pop()!; + if (visitedDirectories.has(directory.ino)) throw new SFSError(EIO); + visitedDirectories.add(directory.ino); + + const inoOff = this.inodeOffset(directory.ino); + if ((this.r32(inoOff + INO_MODE) & S_IFMT) !== S_IFDIR) { + throw new SFSError(EIO); + } + const dirSize = this.r64(inoOff + INO_SIZE); + let pos = 0; + while (pos < dirSize) { + const fileBlock = Math.floor(pos / BLOCK_SIZE); + const blockOff = pos % BLOCK_SIZE; + const phys = this.inodeBlockMap(directory.ino, fileBlock, false); + if (phys <= 0) throw new SFSError(EIO); + const blockBase = phys * BLOCK_SIZE; + const remain = Math.min(dirSize - pos, BLOCK_SIZE - blockOff); + + let off = blockOff; + while (off < blockOff + remain) { + const abs = blockBase + off; + const entIno = this.r32(abs); + const recLen = this.view.getUint16(abs + 4, true); + const nameLen = this.view.getUint16(abs + 6, true); + if (!this.isValidDirEntry(off, blockOff + remain, recLen, nameLen)) { + throw new SFSError(EIO); + } + if (entIno !== 0) { + if (!this.inodeIsAllocated(entIno)) throw new SFSError(EIO); + const name = safeDecode( + this.u8.subarray( + abs + DIRENT_HEADER_SIZE, + abs + DIRENT_HEADER_SIZE + nameLen, + ), + ); + if (name !== "." && name !== "..") { + const childPath = + directory.path === "/" + ? `/${name}` + : `${directory.path}/${name}`; + const childOff = this.inodeOffset(entIno); + const generation = this.r64(childOff + INO_GENERATION); + const key = `${entIno}:${generation}`; + let identity = identities.get(key); + if (!identity) { + identity = { + ino: entIno, + generation, + dataSequence: + Atomics.load( + this.i32, + (childOff + INO_DATA_SEQUENCE) >> 2, + ) >>> 0, + paths: [], + }; + identities.set(key, identity); + } + identity.paths.push(childPath); + if ((this.r32(childOff + INO_MODE) & S_IFMT) === S_IFDIR) { + directories.push({ ino: entIno, path: childPath }); + } + } + } + off += recLen; + } + pos += remain; + } + } + return identities; + } + statfs(): SharedFsStats { const blockSize = this.r32(SB_BLOCK_SIZE); const currentBlocks = this.r32(SB_TOTAL_BLOCKS); @@ -370,6 +586,60 @@ export class SharedFS { this.view.setBigUint64(off, BigInt(v), true); } + /** + * Wait for a shared lock word to change. + * + * Browser main threads forbid Atomics.wait(). The legacy shared-filesystem + * BrowserKernel path still exposes synchronous MemoryFileSystem methods on + * that thread, so fall back to atomic polling while the independently + * scheduled kernel worker finishes its short critical section. Preferred + * kernel-owned browser boots never exercise this fallback. + */ + private waitForAtomicChange(index: number, expected: number): void { + if (this.atomicsWaitAllowed !== false) { + try { + Atomics.wait(this.i32, index, expected); + this.atomicsWaitAllowed = true; + return; + } catch (error) { + if (!(error instanceof TypeError)) throw error; + this.atomicsWaitAllowed = false; + } + } + while (Atomics.load(this.i32, index) === expected) { + // Synchronous SharedFS cannot yield a Promise here. The lock owner is a + // different worker, so atomic polling does not prevent its progress. + } + } + + private resetAllocationHints(): void { + this.blockAllocHint = this.findNextFreeBlockHint(); + this.inodeAllocHint = this.findNextFreeInodeHint(); + } + + private findNextFreeBlockHint(): number { + const totalBlocks = this.r32(SB_TOTAL_BLOCKS); + const dataStart = this.r32(SB_DATA_START); + const bbStart = this.r32(SB_BLOCK_BITMAP_START) * BLOCK_SIZE; + for (let blockNo = dataStart; blockNo < totalBlocks; blockNo++) { + const idx = (bbStart >> 2) + (blockNo >> 5); + const bit = blockNo & 31; + if ((Atomics.load(this.i32, idx) & (1 << bit)) === 0) return blockNo; + } + return dataStart; + } + + private findNextFreeInodeHint(): number { + const totalInodes = this.r32(SB_TOTAL_INODES); + const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; + for (let ino = 2; ino < totalInodes; ino++) { + const idx = (ibStart >> 2) + (ino >> 5); + const bit = ino & 31; + if ((Atomics.load(this.i32, idx) & (1 << bit)) === 0) return ino; + } + return 2; + } + // ── Superblock lock (for grow) ─────────────────────────────────── private sbLock(): void { @@ -377,7 +647,7 @@ export class SharedFS { for (;;) { const old = Atomics.compareExchange(this.i32, idx, 0, 1); if (old === 0) return; - Atomics.wait(this.i32, idx, 1); + this.waitForAtomicChange(idx, 1); } } @@ -387,36 +657,100 @@ export class SharedFS { Atomics.notify(this.i32, idx, Infinity); } + // ── Namespace lock (path resolution and mutation) ─────────────── + + private namespaceLock(): void { + const idx = SB_NAMESPACE_LOCK >> 2; + for (;;) { + const old = Atomics.compareExchange(this.i32, idx, 0, 1); + if (old === 0) return; + this.waitForAtomicChange(idx, 1); + } + } + + private namespaceUnlock(): void { + const idx = SB_NAMESPACE_LOCK >> 2; + Atomics.store(this.i32, idx, 0); + Atomics.notify(this.i32, idx, Infinity); + } + + private withNamespaceLock(operation: () => T): T { + this.namespaceLock(); + try { + return operation(); + } finally { + this.namespaceUnlock(); + } + } + + /** Reset process-local runtime state after copying a portable image. */ + private resetRestoredRuntimeState(): void { + // The buffer is private to fromImage(), so stale locks can be cleared + // directly before any lock-taking operation is attempted. + Atomics.store(this.i32, SB_GLOBAL_LOCK >> 2, 0); + Atomics.store(this.i32, SB_NAMESPACE_LOCK >> 2, 0); + this.u8.fill(0, FD_TABLE_OFFSET, BLOCK_SIZE); + + const totalInodes = this.r32(SB_TOTAL_INODES); + const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; + for (let ino = 0; ino < totalInodes; ino++) { + const off = this.inodeOffset(ino); + this.w32(off + INO_LOCK_STATE, 0); + this.w32(off + INO_OPEN_COUNT, 0); + if (ino < 2) continue; + const word = this.r32(ibStart + (ino >> 5) * 4); + if ((word & (1 << (ino & 31))) === 0) continue; + if (this.r32(off + INO_LINK_COUNT) !== 0) continue; + + const mode = this.r32(off + INO_MODE); + const size = this.r64(off + INO_SIZE); + if ((mode & S_IFMT) === S_IFLNK && size <= INLINE_SYMLINK_SIZE) { + this.u8.fill( + 0, + off + INO_DIRECT, + off + INO_DIRECT + INLINE_SYMLINK_SIZE, + ); + this.w64(off + INO_SIZE, 0); + } else { + this.inodeTruncate(ino, 0); + } + this.inodeFree(ino); + } + } + // ── Block allocator ────────────────────────────────────────────── private blockAlloc(): number { const totalBlocks = this.r32(SB_TOTAL_BLOCKS); const bbStart = this.r32(SB_BLOCK_BITMAP_START) * BLOCK_SIZE; - const numWords = Math.ceil(totalBlocks / 32); - - for (let w = 0; w < numWords; w++) { - const idx = (bbStart >> 2) + w; + const dataStart = this.r32(SB_DATA_START); + const start = + this.blockAllocHint >= dataStart && this.blockAllocHint < totalBlocks + ? this.blockAllocHint + : dataStart; + const allocatableBlocks = totalBlocks - dataStart; + + for (let checked = 0; checked < allocatableBlocks; checked++) { + const blockNo = + dataStart + ((start - dataStart + checked) % allocatableBlocks); + const idx = (bbStart >> 2) + (blockNo >> 5); + const bit = blockNo & 31; const word = Atomics.load(this.i32, idx); - if (word === -1) continue; // all bits set (0xFFFFFFFF as int32) - - for (let bit = 0; bit < 32; bit++) { - const blockNo = w * 32 + bit; - if (blockNo >= totalBlocks) return ENOSPC; - if (word & (1 << bit)) continue; - - const desired = word | (1 << bit); - const old = Atomics.compareExchange(this.i32, idx, word, desired); - if (old === word) { - Atomics.sub(this.i32, SB_FREE_BLOCKS >> 2, 1); - // Zero the newly allocated block - const off = blockNo * BLOCK_SIZE; - this.u8.fill(0, off, off + BLOCK_SIZE); - return blockNo; - } - // CAS failed — retry this word - w--; - break; + if (word & (1 << bit)) continue; + + const desired = word | (1 << bit); + const old = Atomics.compareExchange(this.i32, idx, word, desired); + if (old === word) { + Atomics.sub(this.i32, SB_FREE_BLOCKS >> 2, 1); + this.blockAllocHint = + blockNo + 1 < totalBlocks ? blockNo + 1 : dataStart; + // Zero the newly allocated block + const off = blockNo * BLOCK_SIZE; + this.u8.fill(0, off, off + BLOCK_SIZE); + return blockNo; } + // CAS failed — retry this candidate. + checked--; } return ENOSPC; } @@ -442,6 +776,9 @@ export class SharedFS { if (old === word) break; } Atomics.add(this.i32, SB_FREE_BLOCKS >> 2, 1); + if (blockNo >= this.r32(SB_DATA_START) && blockNo < this.blockAllocHint) { + this.blockAllocHint = blockNo; + } } // ── Growth ─────────────────────────────────────────────────────── @@ -478,6 +815,7 @@ export class SharedFS { this.w32(SB_TOTAL_BLOCKS, newTotal); Atomics.add(this.i32, SB_FREE_BLOCKS >> 2, growBy); Atomics.add(this.i32, SB_GENERATION >> 2, 1); + this.blockAllocHint = current; return 0; } finally { this.sbUnlock(); @@ -496,34 +834,40 @@ export class SharedFS { private inodeAlloc(): number { const totalInodes = this.r32(SB_TOTAL_INODES); const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; - const numWords = Math.ceil(totalInodes / 32); - - for (let w = 0; w < numWords; w++) { - const idx = (ibStart >> 2) + w; + const start = + this.inodeAllocHint >= 2 && this.inodeAllocHint < totalInodes + ? this.inodeAllocHint + : 2; + const allocatableInodes = totalInodes - 2; + + for (let checked = 0; checked < allocatableInodes; checked++) { + const ino = 2 + ((start - 2 + checked) % allocatableInodes); + const idx = (ibStart >> 2) + (ino >> 5); + const bit = ino & 31; const word = Atomics.load(this.i32, idx); - if (word === -1) continue; - - for (let bit = 0; bit < 32; bit++) { - const ino = w * 32 + bit; - if (ino >= totalInodes) return ENOSPC; - if (word & (1 << bit)) continue; - - const desired = word | (1 << bit); - const old = Atomics.compareExchange(this.i32, idx, word, desired); - if (old === word) { - Atomics.sub(this.i32, SB_FREE_INODES >> 2, 1); - // Zero the inode - const off = this.inodeOffset(ino); - this.u8.fill(0, off, off + INODE_SIZE); - return ino; - } - w--; - break; + if (word & (1 << bit)) continue; + + const desired = word | (1 << bit); + const old = Atomics.compareExchange(this.i32, idx, word, desired); + if (old === word) { + Atomics.sub(this.i32, SB_FREE_INODES >> 2, 1); + this.inodeAllocHint = ino + 1 < totalInodes ? ino + 1 : 2; + // Zero the inode + const off = this.inodeOffset(ino); + this.u8.fill(0, off, off + INODE_SIZE); + this.w64(off + INO_GENERATION, this.nextInodeGeneration()); + return ino; } + // CAS failed — retry this candidate. + checked--; } return ENOSPC; } + private nextInodeGeneration(): number { + return Atomics.add(this.i32, SB_GENERATION >> 2, 1) + 1; + } + private inodeFree(ino: number): void { const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; const idx = (ibStart >> 2) + (ino >> 5); @@ -531,11 +875,74 @@ export class SharedFS { for (;;) { const word = Atomics.load(this.i32, idx); + if ((word & (1 << bit)) === 0) throw new SFSError(EIO); const desired = word & ~(1 << bit); const old = Atomics.compareExchange(this.i32, idx, word, desired); if (old === word) break; } Atomics.add(this.i32, SB_FREE_INODES >> 2, 1); + if (ino >= 2 && ino < this.inodeAllocHint) this.inodeAllocHint = ino; + } + + private inodeAddOpenRef(ino: number): boolean { + this.inodeWriteLock(ino); + try { + const off = this.inodeOffset(ino); + if (this.r32(off + INO_LINK_COUNT) === 0) return false; + this.w32(off + INO_OPEN_COUNT, this.r32(off + INO_OPEN_COUNT) + 1); + return true; + } finally { + this.inodeWriteUnlock(ino); + } + } + + private inodeDropOpenRef(ino: number): void { + let shouldFree = false; + this.inodeWriteLock(ino); + try { + const off = this.inodeOffset(ino); + const openCount = this.r32(off + INO_OPEN_COUNT); + if (openCount > 0) { + this.w32(off + INO_OPEN_COUNT, openCount - 1); + } + if (openCount <= 1 && this.r32(off + INO_LINK_COUNT) === 0) { + this.inodeTruncate(ino, 0); + shouldFree = true; + } + } finally { + this.inodeWriteUnlock(ino); + } + if (shouldFree) this.inodeFree(ino); + } + + private inodeDropLinkRefLocked(ino: number): boolean { + const off = this.inodeOffset(ino); + const linkCount = this.r32(off + INO_LINK_COUNT); + if (linkCount > 1) { + this.w32(off + INO_LINK_COUNT, linkCount - 1); + this.w64(off + INO_CTIME, Date.now()); + return false; + } + return this.inodeOrphanLocked(ino); + } + + private inodeOrphanLocked(ino: number): boolean { + const off = this.inodeOffset(ino); + this.w32(off + INO_LINK_COUNT, 0); + this.w64(off + INO_CTIME, Date.now()); + if (this.r32(off + INO_OPEN_COUNT) > 0) return false; + const mode = this.r32(off + INO_MODE); + const size = this.r64(off + INO_SIZE); + if ((mode & S_IFMT) === S_IFLNK && size <= INLINE_SYMLINK_SIZE) { + // Short symlink targets are stored inline in the inode's direct-pointer + // area. POSIX unlink removes the symlink inode itself even if the target + // is dangling; do not interpret inline target bytes as block numbers. + this.u8.fill(0, off + INO_DIRECT, off + INO_DIRECT + INLINE_SYMLINK_SIZE); + this.w64(off + INO_SIZE, 0); + } else { + this.inodeTruncate(ino, 0); + } + return true; } // ── Inode locking ──────────────────────────────────────────────── @@ -545,15 +952,10 @@ export class SharedFS { for (;;) { const cur = Atomics.load(this.i32, lockIdx); if (cur & WRITER_BIT) { - Atomics.wait(this.i32, lockIdx, cur); + this.waitForAtomicChange(lockIdx, cur); continue; } - const old = Atomics.compareExchange( - this.i32, - lockIdx, - cur, - cur + 1, - ); + const old = Atomics.compareExchange(this.i32, lockIdx, cur, cur + 1); if (old === cur) return; } } @@ -571,15 +973,10 @@ export class SharedFS { for (;;) { const cur = Atomics.load(this.i32, lockIdx); if (cur !== 0) { - Atomics.wait(this.i32, lockIdx, cur); + this.waitForAtomicChange(lockIdx, cur); continue; } - const old = Atomics.compareExchange( - this.i32, - lockIdx, - 0, - WRITER_BIT, - ); + const old = Atomics.compareExchange(this.i32, lockIdx, 0, WRITER_BIT); if (old === 0) return; } } @@ -614,18 +1011,26 @@ export class SharedFS { fileBlock -= DIRECT_BLOCKS; if (fileBlock < PTRS_PER_BLOCK) { let ind = this.r32(inoOff + INO_INDIRECT); + let allocatedIndirect = false; if (ind === 0) { if (!allocate) return 0; ind = this.blockAllocWithGrow(); if (ind < 0) return ind; this.w32(inoOff + INO_INDIRECT, ind); + allocatedIndirect = true; } const ptrOff = ind * BLOCK_SIZE + fileBlock * 4; const ptr = this.r32(ptrOff); if (ptr !== 0) return ptr; if (!allocate) return 0; const blk = this.blockAllocWithGrow(); - if (blk < 0) return blk; + if (blk < 0) { + if (allocatedIndirect) { + this.w32(inoOff + INO_INDIRECT, 0); + this.blockFree(ind); + } + return blk; + } this.w32(ptrOff, blk); return blk; } @@ -637,20 +1042,30 @@ export class SharedFS { const idx2 = fileBlock % PTRS_PER_BLOCK; let dind = this.r32(inoOff + INO_DOUBLE_INDIRECT); + let allocatedDoubleIndirect = false; if (dind === 0) { if (!allocate) return 0; dind = this.blockAllocWithGrow(); if (dind < 0) return dind; this.w32(inoOff + INO_DOUBLE_INDIRECT, dind); + allocatedDoubleIndirect = true; } const l1Off = dind * BLOCK_SIZE + idx1 * 4; let l1 = this.r32(l1Off); + let allocatedFirstLevel = false; if (l1 === 0) { if (!allocate) return 0; l1 = this.blockAllocWithGrow(); - if (l1 < 0) return l1; + if (l1 < 0) { + if (allocatedDoubleIndirect) { + this.w32(inoOff + INO_DOUBLE_INDIRECT, 0); + this.blockFree(dind); + } + return l1; + } this.w32(l1Off, l1); + allocatedFirstLevel = true; } const l2Off = l1 * BLOCK_SIZE + idx2 * 4; @@ -658,7 +1073,17 @@ export class SharedFS { if (ptr !== 0) return ptr; if (!allocate) return 0; const blk = this.blockAllocWithGrow(); - if (blk < 0) return blk; + if (blk < 0) { + if (allocatedFirstLevel) { + this.w32(l1Off, 0); + this.blockFree(l1); + } + if (allocatedDoubleIndirect) { + this.w32(inoOff + INO_DOUBLE_INDIRECT, 0); + this.blockFree(dind); + } + return blk; + } this.w32(l2Off, blk); return blk; } @@ -711,6 +1136,11 @@ export class SharedFS { count: number, ): number { const inoOff = this.inodeOffset(ino); + const size = this.r64(inoOff + INO_SIZE); + if (offset > size) { + this.zeroOldEofTail(ino, size); + } + let totalWritten = 0; let srcPos = 0; @@ -721,7 +1151,10 @@ export class SharedFS { if (chunk > count) chunk = count; const phys = this.inodeBlockMap(ino, fileBlock, true); - if (phys < 0) return totalWritten > 0 ? totalWritten : phys; + if (phys < 0) { + if (totalWritten === 0) return phys; + break; + } const dstOff = phys * BLOCK_SIZE + blockOff; this.u8.set(src.subarray(srcPos, srcPos + chunk), dstOff); @@ -732,13 +1165,47 @@ export class SharedFS { totalWritten += chunk; } - const size = this.r64(inoOff + INO_SIZE); - if (offset > size) { + if (totalWritten > 0 && offset > this.r64(inoOff + INO_SIZE)) { this.w64(inoOff + INO_SIZE, offset); } + if (totalWritten > 0) { + const now = Date.now(); + this.w64(inoOff + INO_MTIME, now); + this.w64(inoOff + INO_CTIME, now); + Atomics.add(this.i32, (inoOff + INO_DATA_SEQUENCE) >> 2, 1); + } return totalWritten; } + private zeroInodeRange(ino: number, start: number, end: number): void { + while (start < end) { + const fileBlock = Math.floor(start / BLOCK_SIZE); + const blockOff = start % BLOCK_SIZE; + const chunk = Math.min(BLOCK_SIZE - blockOff, end - start); + const phys = this.inodeBlockMap(ino, fileBlock, false); + if (phys > 0) { + const abs = phys * BLOCK_SIZE + blockOff; + this.u8.fill(0, abs, abs + chunk); + } + start += chunk; + } + } + + /** + * Zero only the allocated tail of the old EOF block. Sparse extension does + * not need to walk logical holes: absent blocks already read as zero and a + * newly allocated block is cleared by blockAlloc(). + */ + private zeroOldEofTail(ino: number, oldSize: number): void { + const blockOff = oldSize % BLOCK_SIZE; + if (blockOff === 0) return; + const fileBlock = Math.floor(oldSize / BLOCK_SIZE); + const phys = this.inodeBlockMap(ino, fileBlock, false); + if (phys <= 0) return; + const start = phys * BLOCK_SIZE + blockOff; + this.u8.fill(0, start, phys * BLOCK_SIZE + BLOCK_SIZE); + } + private freeBlocksFrom(ino: number, fromBlock: number): void { const inoOff = this.inodeOffset(ino); @@ -754,8 +1221,7 @@ export class SharedFS { // Single indirect const ind = this.r32(inoOff + INO_INDIRECT); if (ind) { - const start = - fromBlock > DIRECT_BLOCKS ? fromBlock - DIRECT_BLOCKS : 0; + const start = fromBlock > DIRECT_BLOCKS ? fromBlock - DIRECT_BLOCKS : 0; for (let i = start; i < PTRS_PER_BLOCK; i++) { const ptrOff = ind * BLOCK_SIZE + i * 4; const ptr = this.r32(ptrOff); @@ -805,59 +1271,439 @@ export class SharedFS { } } - private inodeTruncate(ino: number, newSize: number): void { + private inodeTruncate( + ino: number, + newSize: number, + forceDataMutation = false, + ): void { const inoOff = this.inodeOffset(ino); const curSize = this.r64(inoOff + INO_SIZE); + const sizeChanged = newSize !== curSize; if (newSize >= curSize) { + if (newSize > curSize) { + this.zeroOldEofTail(ino, curSize); + } this.w64(inoOff + INO_SIZE, newSize); + if (sizeChanged || forceDataMutation) { + const now = Date.now(); + this.w64(inoOff + INO_MTIME, now); + this.w64(inoOff + INO_CTIME, now); + Atomics.add(this.i32, (inoOff + INO_DATA_SEQUENCE) >> 2, 1); + } return; } + if (newSize % BLOCK_SIZE !== 0) { + this.zeroInodeRange( + ino, + newSize, + Math.ceil(newSize / BLOCK_SIZE) * BLOCK_SIZE, + ); + } const keepBlocks = Math.ceil(newSize / BLOCK_SIZE); this.freeBlocksFrom(ino, keepBlocks); this.w64(inoOff + INO_SIZE, newSize); + if (sizeChanged || forceDataMutation) { + const now = Date.now(); + this.w64(inoOff + INO_MTIME, now); + this.w64(inoOff + INO_CTIME, now); + Atomics.add(this.i32, (inoOff + INO_DATA_SEQUENCE) >> 2, 1); + } + } + + private validateFileSize(size: number): void { + if (!Number.isSafeInteger(size) || size < 0) throw new SFSError(EINVAL); + if (size > MAX_FILE_SIZE) throw new SFSError(EFBIG); } // ── Directory operations ───────────────────────────────────────── + private touchDirectoryMutation(dirIno: number): void { + const inoOff = this.inodeOffset(dirIno); + const now = Date.now(); + this.w64(inoOff + INO_MTIME, now); + this.w64(inoOff + INO_CTIME, now); + const mutationSequence = + (Atomics.add(this.i32, (inoOff + INO_DIR_SEQUENCE) >> 2, 1) + 1) >>> 0; + const index = this.dirIndexes.get(dirIno); + if (index) { + index.mutationSequence = mutationSequence; + index.size = this.r64(inoOff + INO_SIZE); + } + } + + private dirNameKey(name: Uint8Array): string { + return safeDecode(name); + } + + private dirEntryNameMatches(abs: number, name: Uint8Array): boolean { + const entNameLen = this.view.getUint16(abs + 6, true); + if (entNameLen !== name.length) return false; + for (let i = 0; i < name.length; i++) { + if (this.u8[abs + DIRENT_HEADER_SIZE + i] !== name[i]) return false; + } + return true; + } + + private isValidDirEntry( + off: number, + endOff: number, + recLen: number, + nameLen: number, + ): boolean { + return ( + recLen >= DIRENT_HEADER_SIZE && + recLen % 4 === 0 && + off + recLen <= endOff && + nameLen <= recLen - DIRENT_HEADER_SIZE + ); + } + + private inodeIsAllocated(ino: number): boolean { + const totalInodes = this.r32(SB_TOTAL_INODES); + if (ino <= 0 || ino >= totalInodes) return false; + const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; + const word = Atomics.load(this.i32, (ibStart >> 2) + (ino >> 5)); + return (word & (1 << (ino & 31))) !== 0; + } + + private rebuildDirIndex( + dirIno: number, + generation: number, + mutationSequence: number, + dirSize: number, + ): DirIndex | number { + const entries = new Map(); + const free: Array<{ abs: number; recLen: number }> = []; + let pos = 0; + + while (pos < dirSize) { + const fileBlock = Math.floor(pos / BLOCK_SIZE); + const blockOff = pos % BLOCK_SIZE; + const phys = this.inodeBlockMap(dirIno, fileBlock, false); + if (phys <= 0) return EIO; + + const blockBase = phys * BLOCK_SIZE; + let remain = dirSize - pos; + if (remain > BLOCK_SIZE - blockOff) remain = BLOCK_SIZE - blockOff; + + let off = blockOff; + while (off < blockOff + remain) { + const abs = blockBase + off; + const entIno = this.r32(abs); + const recLen = this.view.getUint16(abs + 4, true); + const entNameLen = this.view.getUint16(abs + 6, true); + + if (!this.isValidDirEntry(off, blockOff + remain, recLen, entNameLen)) + return EIO; + + if (entIno !== 0) { + if (!this.inodeIsAllocated(entIno)) return EIO; + const name = safeDecode( + this.u8.subarray( + abs + DIRENT_HEADER_SIZE, + abs + DIRENT_HEADER_SIZE + entNameLen, + ), + ); + entries.set(name, { + ino: entIno, + abs, + recLen, + nameLen: entNameLen, + }); + } else if (recLen >= DIRENT_HEADER_SIZE) { + free.push({ abs, recLen }); + } + + off += recLen; + } + pos += remain; + } + + const index = { + generation, + mutationSequence, + size: dirSize, + entries, + free, + }; + this.dirIndexes.set(dirIno, index); + return index; + } + + private getDirIndex(dirIno: number): DirIndex | null | number { + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + const generation = this.r64(inoOff + INO_GENERATION); + const mutationSequence = + Atomics.load(this.i32, (inoOff + INO_DIR_SEQUENCE) >> 2) >>> 0; + const cached = this.dirIndexes.get(dirIno); + if ( + cached && + cached.generation === generation && + cached.mutationSequence === mutationSequence && + cached.size === dirSize + ) { + return cached; + } + if (cached) this.dirIndexes.delete(dirIno); + + if (dirSize < SharedFS.DIR_INDEX_MIN_SIZE) return null; + return this.rebuildDirIndex(dirIno, generation, mutationSequence, dirSize); + } + + private updateDirIndexAdd( + dirIno: number, + name: Uint8Array, + childIno: number, + abs: number, + recLen: number, + ): void { + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + const generation = this.r64(inoOff + INO_GENERATION); + const index = this.dirIndexes.get(dirIno); + if (!index) return; + if (index.generation !== generation) { + this.dirIndexes.delete(dirIno); + return; + } + index.size = dirSize; + index.entries.set(this.dirNameKey(name), { + ino: childIno, + abs, + recLen, + nameLen: name.length, + }); + } + + private useDirIndexFreeSlot( + index: DirIndex, + dirIno: number, + name: Uint8Array, + childIno: number, + ): boolean { + const needed = align4(DIRENT_HEADER_SIZE + name.length); + + for (let i = index.free.length - 1; i >= 0; i--) { + const slot = index.free[i]; + if (slot.recLen < needed) continue; + index.free.splice(i, 1); + if ( + this.r32(slot.abs) !== 0 || + this.view.getUint16(slot.abs + 4, true) !== slot.recLen + ) { + continue; + } + + this.w32(slot.abs, childIno); + this.view.setUint16(slot.abs + 6, name.length, true); + this.u8.set(name, slot.abs + DIRENT_HEADER_SIZE); + this.touchDirectoryMutation(dirIno); + this.updateDirIndexAdd(dirIno, name, childIno, slot.abs, slot.recLen); + return true; + } + + return false; + } + + private updateDirIndexRemove(dirIno: number, name: Uint8Array): void { + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + const generation = this.r64(inoOff + INO_GENERATION); + const index = this.dirIndexes.get(dirIno); + if (!index) return; + if (index.generation !== generation || index.size !== dirSize) { + this.dirIndexes.delete(dirIno); + return; + } + index.entries.delete(this.dirNameKey(name)); + } + + private updateDirIndexRecLen( + dirIno: number, + abs: number, + recLen: number, + ): void { + const index = this.dirIndexes.get(dirIno); + if (!index) return; + for (const entry of index.entries.values()) { + if (entry.abs === abs) { + entry.recLen = recLen; + return; + } + } + } + private dirLookup(dirIno: number, name: Uint8Array): number { + const index = this.getDirIndex(dirIno); + if (typeof index === "number") return index; + if (index) { + const entry = index.entries.get(this.dirNameKey(name)); + if (!entry) return ENOENT; + + // Validate positive hits against the backing directory entry so stale + // in-process indexes cannot resurrect an externally removed name. + if ( + this.r32(entry.abs) === entry.ino && + this.inodeIsAllocated(entry.ino) && + this.view.getUint16(entry.abs + 4, true) === entry.recLen && + this.view.getUint16(entry.abs + 6, true) === entry.nameLen && + this.dirEntryNameMatches(entry.abs, name) + ) { + return entry.ino; + } + + index.entries.delete(this.dirNameKey(name)); + return ENOENT; + } + const inoOff = this.inodeOffset(dirIno); const dirSize = this.r64(inoOff + INO_SIZE); let pos = 0; - while (pos < dirSize) { - const fileBlock = Math.floor(pos / BLOCK_SIZE); - const blockOff = pos % BLOCK_SIZE; - const phys = this.inodeBlockMap(dirIno, fileBlock, false); + while (pos < dirSize) { + const fileBlock = Math.floor(pos / BLOCK_SIZE); + const blockOff = pos % BLOCK_SIZE; + const phys = this.inodeBlockMap(dirIno, fileBlock, false); + if (phys <= 0) return EIO; + + const blockBase = phys * BLOCK_SIZE; + let remain = dirSize - pos; + if (remain > BLOCK_SIZE - blockOff) remain = BLOCK_SIZE - blockOff; + + let off = blockOff; + while (off < blockOff + remain) { + const abs = blockBase + off; + const entIno = this.r32(abs); + const recLen = this.view.getUint16(abs + 4, true); + const entNameLen = this.view.getUint16(abs + 6, true); + + if (!this.isValidDirEntry(off, blockOff + remain, recLen, entNameLen)) + return EIO; + + if (entIno !== 0 && entNameLen === name.length) { + let match = true; + for (let i = 0; i < name.length; i++) { + if (this.u8[abs + DIRENT_HEADER_SIZE + i] !== name[i]) { + match = false; + break; + } + } + if (match) { + if (!this.inodeIsAllocated(entIno)) return EIO; + return entIno; + } + } + off += recLen; + } + pos += remain; + } + return ENOENT; + } + + private findLastDirEntryInBlock( + dirIno: number, + fileBlock: number, + endOff: number, + ): number { + const phys = this.inodeBlockMap(dirIno, fileBlock, false); + if (phys <= 0) return -1; + const blockBase = phys * BLOCK_SIZE; + let off = 0; + let lastAbs = -1; + while (off < endOff) { + const abs = blockBase + off; + const recLen = this.view.getUint16(abs + 4, true); + if ( + recLen < DIRENT_HEADER_SIZE || + recLen % 4 !== 0 || + off + recLen > endOff + ) { + return -1; + } + lastAbs = abs; + off += recLen; + } + return off === endOff ? lastAbs : -1; + } + + private dirAppendEntry( + dirIno: number, + name: Uint8Array, + childIno: number, + lastEntAbs = -1, + ): number { + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + const needed = align4(DIRENT_HEADER_SIZE + name.length); + + // No space found — append a new entry at the end. + // Directory entries must not cross block boundaries (like ext2). + let appendPos = dirSize; + let fileBlock = Math.floor(appendPos / BLOCK_SIZE); + let blockOff = appendPos % BLOCK_SIZE; + let reservedPhys = 0; + + if (blockOff !== 0 && blockOff + needed > BLOCK_SIZE) { + // Entry doesn't fit in remaining space — skip to next block. + const gap = BLOCK_SIZE - blockOff; + let padPhys = 0; + if (gap >= DIRENT_HEADER_SIZE) { + padPhys = this.inodeBlockMap(dirIno, fileBlock, false); + if (padPhys <= 0) return EIO; + } else { + if (lastEntAbs < 0) { + lastEntAbs = this.findLastDirEntryInBlock( + dirIno, + fileBlock, + blockOff, + ); + } + if (lastEntAbs < 0) return EIO; + } + + // Reserve the destination block before changing the old tail. If the + // allocation fails, the directory remains byte-for-byte unchanged. + reservedPhys = this.inodeBlockMap(dirIno, fileBlock + 1, true); + if (reservedPhys < 0) return reservedPhys; + if (gap >= DIRENT_HEADER_SIZE) { + // Write a padding entry (ino=0) to fill the gap + const padAbs = padPhys * BLOCK_SIZE + blockOff; + this.w32(padAbs, 0); + this.view.setUint16(padAbs + 4, gap, true); + this.view.setUint16(padAbs + 6, 0, true); + } else { + // Gap too small for a padding entry — extend last entry's recLen + const oldRecLen = this.view.getUint16(lastEntAbs + 4, true); + const newRecLen = oldRecLen + gap; + this.view.setUint16(lastEntAbs + 4, newRecLen, true); + this.updateDirIndexRecLen(dirIno, lastEntAbs, newRecLen); + } + appendPos = (fileBlock + 1) * BLOCK_SIZE; + fileBlock++; + blockOff = 0; + } + + // Need a new block? + let phys: number; + if (blockOff === 0) { + phys = reservedPhys || this.inodeBlockMap(dirIno, fileBlock, true); + if (phys < 0) return phys; + } else { + phys = this.inodeBlockMap(dirIno, fileBlock, false); if (phys <= 0) return EIO; + } - const blockBase = phys * BLOCK_SIZE; - let remain = dirSize - pos; - if (remain > BLOCK_SIZE - blockOff) remain = BLOCK_SIZE - blockOff; - - let off = blockOff; - while (off < blockOff + remain) { - const abs = blockBase + off; - const entIno = this.r32(abs); - const recLen = this.view.getUint16(abs + 4, true); - const entNameLen = this.view.getUint16(abs + 6, true); - - if (recLen === 0) return EIO; + const abs = phys * BLOCK_SIZE + blockOff; + this.w32(abs, childIno); + this.view.setUint16(abs + 4, needed, true); + this.view.setUint16(abs + 6, name.length, true); + this.u8.set(name, abs + DIRENT_HEADER_SIZE); - if (entIno !== 0 && entNameLen === name.length) { - let match = true; - for (let i = 0; i < name.length; i++) { - if (this.u8[abs + DIRENT_HEADER_SIZE + i] !== name[i]) { - match = false; - break; - } - } - if (match) return entIno; - } - off += recLen; - } - pos += remain; - } - return ENOENT; + this.w64(inoOff + INO_SIZE, appendPos + needed); + this.touchDirectoryMutation(dirIno); + this.updateDirIndexAdd(dirIno, name, childIno, abs, needed); + return 0; } private dirAddEntry( @@ -865,6 +1711,16 @@ export class SharedFS { name: Uint8Array, childIno: number, ): number { + const index = this.getDirIndex(dirIno); + if (typeof index === "number") return index; + if (index) { + if (this.useDirIndexFreeSlot(index, dirIno, name, childIno)) return 0; + + // When no indexed deleted slot is available, append instead of scanning + // every existing record again solely to discover internal slack. + return this.dirAppendEntry(dirIno, name, childIno); + } + const inoOff = this.inodeOffset(dirIno); const dirSize = this.r64(inoOff + INO_SIZE); const needed = align4(DIRENT_HEADER_SIZE + name.length); @@ -891,13 +1747,21 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return EIO; + if ( + recLen < DIRENT_HEADER_SIZE || + recLen % 4 !== 0 || + off + recLen > blockOff + remain || + entNameLen > recLen - DIRENT_HEADER_SIZE + ) + return EIO; if (entIno === 0 && recLen >= needed) { // Reuse deleted entry this.w32(abs, childIno); this.view.setUint16(abs + 6, name.length, true); this.u8.set(name, abs + DIRENT_HEADER_SIZE); + this.touchDirectoryMutation(dirIno); + this.updateDirIndexAdd(dirIno, name, childIno, abs, recLen); return 0; } @@ -912,6 +1776,8 @@ export class SharedFS { this.view.setUint16(newAbs + 4, slack, true); this.view.setUint16(newAbs + 6, name.length, true); this.u8.set(name, newAbs + DIRENT_HEADER_SIZE); + this.touchDirectoryMutation(dirIno); + this.updateDirIndexAdd(dirIno, name, childIno, newAbs, slack); return 0; } @@ -921,55 +1787,108 @@ export class SharedFS { pos += remain; } - // No space found — append a new entry at the end. - // Directory entries must not cross block boundaries (like ext2). - let appendPos = dirSize; - let fileBlock = Math.floor(appendPos / BLOCK_SIZE); - let blockOff = appendPos % BLOCK_SIZE; + return this.dirAppendEntry(dirIno, name, childIno, lastEntAbs); + } - if (blockOff !== 0 && blockOff + needed > BLOCK_SIZE) { - // Entry doesn't fit in remaining space — skip to next block. - const gap = BLOCK_SIZE - blockOff; - if (gap >= DIRENT_HEADER_SIZE) { - // Write a padding entry (ino=0) to fill the gap - const padPhys = this.inodeBlockMap(dirIno, fileBlock, false); - if (padPhys > 0) { - const padAbs = padPhys * BLOCK_SIZE + blockOff; - this.w32(padAbs, 0); - this.view.setUint16(padAbs + 4, gap, true); - this.view.setUint16(padAbs + 6, 0, true); - } - } else if (lastEntAbs >= 0) { - // Gap too small for a padding entry — extend last entry's recLen - const oldRecLen = this.view.getUint16(lastEntAbs + 4, true); - this.view.setUint16(lastEntAbs + 4, oldRecLen + gap, true); + private dirRemoveEntry(dirIno: number, name: Uint8Array): number { + const index = this.getDirIndex(dirIno); + if (typeof index === "number") return index; + if (index) { + const key = this.dirNameKey(name); + const entry = index.entries.get(key); + if (!entry) return ENOENT; + + if ( + this.r32(entry.abs) === entry.ino && + this.view.getUint16(entry.abs + 4, true) === entry.recLen && + this.view.getUint16(entry.abs + 6, true) === entry.nameLen && + this.dirEntryNameMatches(entry.abs, name) + ) { + this.w32(entry.abs, 0); // mark as deleted + index.entries.delete(key); + index.free.push({ abs: entry.abs, recLen: entry.recLen }); + this.touchDirectoryMutation(dirIno); + return 0; } - appendPos = (fileBlock + 1) * BLOCK_SIZE; - fileBlock++; - blockOff = 0; + + index.entries.delete(key); + // Fall through to the linear scan below if the cached slot was stale. } - // Need a new block? - let phys: number; - if (blockOff === 0) { - phys = this.inodeBlockMap(dirIno, fileBlock, true); - if (phys < 0) return phys; - } else { - phys = this.inodeBlockMap(dirIno, fileBlock, false); + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + let pos = 0; + + while (pos < dirSize) { + const fileBlock = Math.floor(pos / BLOCK_SIZE); + const blockOff = pos % BLOCK_SIZE; + const phys = this.inodeBlockMap(dirIno, fileBlock, false); if (phys <= 0) return EIO; - } - const abs = phys * BLOCK_SIZE + blockOff; - this.w32(abs, childIno); - this.view.setUint16(abs + 4, needed, true); - this.view.setUint16(abs + 6, name.length, true); - this.u8.set(name, abs + DIRENT_HEADER_SIZE); + const blockBase = phys * BLOCK_SIZE; + let remain = dirSize - pos; + if (remain > BLOCK_SIZE - blockOff) remain = BLOCK_SIZE - blockOff; - this.w64(inoOff + INO_SIZE, appendPos + needed); - return 0; + let off = blockOff; + while (off < blockOff + remain) { + const abs = blockBase + off; + const entIno = this.r32(abs); + const recLen = this.view.getUint16(abs + 4, true); + const entNameLen = this.view.getUint16(abs + 6, true); + + if (!this.isValidDirEntry(off, blockOff + remain, recLen, entNameLen)) + return EIO; + + if (entIno !== 0 && entNameLen === name.length) { + let match = true; + for (let i = 0; i < name.length; i++) { + if (this.u8[abs + DIRENT_HEADER_SIZE + i] !== name[i]) { + match = false; + break; + } + } + if (match) { + this.w32(abs, 0); // mark as deleted + this.touchDirectoryMutation(dirIno); + this.updateDirIndexRemove(dirIno, name); + return 0; + } + } + off += recLen; + } + pos += remain; + } + return ENOENT; } - private dirRemoveEntry(dirIno: number, name: Uint8Array): number { + private dirReplaceEntryIno( + dirIno: number, + name: Uint8Array, + childIno: number, + ): number { + const index = this.getDirIndex(dirIno); + if (typeof index === "number") return index; + if (index) { + const key = this.dirNameKey(name); + const entry = index.entries.get(key); + + if ( + entry && + this.r32(entry.abs) === entry.ino && + this.view.getUint16(entry.abs + 4, true) === entry.recLen && + this.view.getUint16(entry.abs + 6, true) === entry.nameLen && + this.dirEntryNameMatches(entry.abs, name) + ) { + this.w32(entry.abs, childIno); + entry.ino = childIno; + this.touchDirectoryMutation(dirIno); + return 0; + } + + if (entry) index.entries.delete(key); + // Fall through to the linear scan below if the cached slot was stale. + } + const inoOff = this.inodeOffset(dirIno); const dirSize = this.r64(inoOff + INO_SIZE); let pos = 0; @@ -991,7 +1910,8 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return EIO; + if (!this.isValidDirEntry(off, blockOff + remain, recLen, entNameLen)) + return EIO; if (entIno !== 0 && entNameLen === name.length) { let match = true; @@ -1002,7 +1922,9 @@ export class SharedFS { } } if (match) { - this.w32(abs, 0); // mark as deleted + this.w32(abs, childIno); + this.touchDirectoryMutation(dirIno); + this.updateDirIndexAdd(dirIno, name, childIno, abs, recLen); return 0; } } @@ -1022,7 +1944,7 @@ export class SharedFS { const fileBlock = Math.floor(pos / BLOCK_SIZE); const blockOff = pos % BLOCK_SIZE; const phys = this.inodeBlockMap(dirIno, fileBlock, false); - if (phys <= 0) return true; + if (phys <= 0) throw new SFSError(EIO); const blockBase = phys * BLOCK_SIZE; let remain = dirSize - pos; @@ -1035,12 +1957,17 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) break; + if ( + recLen < DIRENT_HEADER_SIZE || + recLen % 4 !== 0 || + off + recLen > blockOff + remain || + entNameLen > recLen - DIRENT_HEADER_SIZE + ) + throw new SFSError(EIO); if (entIno !== 0) { // Skip "." and ".." - if (entNameLen === 1 && this.u8[abs + DIRENT_HEADER_SIZE] === 0x2e) - { + if (entNameLen === 1 && this.u8[abs + DIRENT_HEADER_SIZE] === 0x2e) { off += recLen; continue; } @@ -1061,6 +1988,21 @@ export class SharedFS { return true; } + private dirIsAncestor(ancestorIno: number, dirIno: number): boolean { + let cur = dirIno; + + for (let depth = 0; depth < MAX_SYMLINK_HOPS * 1024; depth++) { + if (cur === ancestorIno) return true; + if (cur === ROOT_INO) return false; + + const parent = this.dirLookup(cur, DOTDOT_BYTES); + if (parent < 0 || parent === cur) throw new SFSError(EIO); + cur = parent; + } + + throw new SFSError(EIO); + } + // ── Path resolution ────────────────────────────────────────────── private pathResolve(path: string, followSymlinks: boolean): number { @@ -1075,12 +2017,17 @@ export class SharedFS { const part = parts[pi]; if (part.length > MAX_NAME) return ENAMETOOLONG; - const inoOff = this.inodeOffset(ino); - const mode = this.r32(inoOff + INO_MODE); - if ((mode & S_IFMT) !== S_IFDIR) return ENOTDIR; - const nameBytes = encoder.encode(part); - const childIno = this.dirLookup(ino, nameBytes); + let childIno: number; + this.inodeReadLock(ino); + try { + const inoOff = this.inodeOffset(ino); + const mode = this.r32(inoOff + INO_MODE); + if ((mode & S_IFMT) !== S_IFDIR) return ENOTDIR; + childIno = this.dirLookup(ino, nameBytes); + } finally { + this.inodeReadUnlock(ino); + } if (childIno < 0) return childIno; // Check if child is symlink @@ -1112,18 +2059,14 @@ export class SharedFS { if (target.startsWith("/")) { // Absolute symlink — restart from root ino = ROOT_INO; - const targetParts = target - .split("/") - .filter((p) => p.length > 0); + const targetParts = target.split("/").filter((p) => p.length > 0); const remaining = parts.slice(pi + 1); parts.length = 0; parts.push(...targetParts, ...remaining); pi = -1; // will be incremented to 0 } else { // Relative symlink — splice into remaining path - const targetParts = target - .split("/") - .filter((p) => p.length > 0); + const targetParts = target.split("/").filter((p) => p.length > 0); const remaining = parts.slice(pi + 1); parts.length = pi; parts.push(...targetParts, ...remaining); @@ -1175,15 +2118,17 @@ export class SharedFS { this.w64(base + FD_OFFSET, 0); this.w32(base + FD_FLAGS, flags); this.w32(base + FD_IS_DIR, isDir ? 1 : 0); + if (!this.inodeAddOpenRef(ino)) { + Atomics.store(this.i32, idx, 0); + return ENOENT; + } return i; } } return EMFILE; } - private fdGet( - fd: number, - ): { + private fdGet(fd: number): { base: number; ino: number; offset: number; @@ -1216,6 +2161,8 @@ export class SharedFS { const off = this.inodeOffset(ino); return { ino, + generation: this.r64(off + INO_GENERATION), + dataSequence: this.r32(off + INO_DATA_SEQUENCE), mode: this.r32(off + INO_MODE), linkCount: this.r32(off + INO_LINK_COUNT), size: this.r64(off + INO_SIZE), @@ -1227,11 +2174,130 @@ export class SharedFS { }; } + private namespaceEntryIdentity(ino: number): NamespaceEntryIdentity { + const off = this.inodeOffset(ino); + return { + ino, + generation: this.r64(off + INO_GENERATION), + linkCount: this.r32(off + INO_LINK_COUNT), + mode: this.r32(off + INO_MODE), + }; + } + // ── Public API: File operations ────────────────────────────────── open(path: string, flags: number, createMode: number = 0o644): number { + return this.withNamespaceLock(() => + this.openUnlocked(path, flags, createMode), + ); + } + + /** + * Atomically create or truncate an empty lazy-file stub and capture the + * exact data identity produced by that truncation. Existing callers rely on + * lazy registration replacing a path, while the inode lock ensures a peer + * write either precedes the truncation or advances the captured sequence + * after it. + */ + createLazyStub(path: string, mode: number): StatResult { + return this.withNamespaceLock(() => { + const fd = this.openUnlocked(path, O_WRONLY | O_CREAT, mode); + try { + const entry = this.fdGet(fd); + if (!entry) throw new SFSError(EBADF); + this.inodeWriteLock(entry.ino); + try { + this.inodeTruncate(entry.ino, 0, true); + return this.buildStat(entry.ino); + } finally { + this.inodeWriteUnlock(entry.ino); + } + } finally { + this.closeUnlocked(fd); + } + }); + } + + /** + * Atomically replace a lazy stub only if the path still names the exact + * inode content generation observed before an asynchronous fetch. + */ + replaceIfIdentity( + path: string, + expectedIno: number, + expectedGeneration: number, + expectedDataSequence: number, + data: Uint8Array, + ): boolean { + return this.withNamespaceLock(() => { + // Lazy access follows symlinks, so conditionally replace the resolved + // regular target while still validating its exact inode/data identity. + const ino = this.pathResolve(path, true); + if (ino < 0 || ino !== expectedIno) return false; + const off = this.inodeOffset(ino); + if ( + this.r64(off + INO_GENERATION) !== expectedGeneration || + this.r32(off + INO_DATA_SEQUENCE) !== expectedDataSequence + ) + return false; + if ((this.r32(off + INO_MODE) & S_IFMT) !== S_IFREG) return false; + this.validateFileSize(data.byteLength); + + this.inodeWriteLock(ino); + try { + // Descriptor-based writes do not take the namespace lock. Revalidate + // after acquiring the inode lock so a concurrent guest mutation that + // won the race cannot be overwritten by stale fetched bytes. + if ( + this.r64(off + INO_GENERATION) !== expectedGeneration || + this.r32(off + INO_DATA_SEQUENCE) !== expectedDataSequence + ) + return false; + // Lazy backing is attached only to an untouched empty stub. Refuse to + // replace any concrete content even if malformed metadata happens to + // carry its current sequence. + if (this.r64(off + INO_SIZE) !== 0) return false; + + const originalMtime = this.r64(off + INO_MTIME); + const originalCtime = this.r64(off + INO_CTIME); + + this.inodeTruncate(ino, 0, true); + const written = + data.byteLength > 0 + ? this.inodeWriteData(ino, 0, data, data.byteLength) + : 0; + if (written !== data.byteLength) { + this.inodeTruncate(ino, 0, true); + Atomics.store( + this.i32, + (off + INO_DATA_SEQUENCE) >> 2, + expectedDataSequence, + ); + this.w64(off + INO_MTIME, originalMtime); + this.w64(off + INO_CTIME, originalCtime); + throw new SFSError(written < 0 ? written : ENOSPC); + } + return true; + } finally { + this.inodeWriteUnlock(ino); + } + }); + } + + private openUnlocked( + path: string, + flags: number, + createMode: number = 0o644, + ): number { const accMode = flags & O_ACCMODE; const creating = (flags & O_CREAT) !== 0; + const exclusive = (flags & O_EXCL) !== 0; + + if (creating && exclusive) { + const existing = this.pathResolve(path, false); + if (existing >= 0) throw new SFSError(EEXIST); + if (existing !== ENOENT) throw new SFSError(existing); + } let ino = this.pathResolve(path, true); @@ -1244,6 +2310,7 @@ export class SharedFS { const nameBytes = encoder.encode(name); const existing = this.dirLookup(parentIno, nameBytes); if (existing >= 0) { + if (exclusive) throw new SFSError(EEXIST); ino = existing; } else { const newIno = this.inodeAlloc(); @@ -1280,7 +2347,7 @@ export class SharedFS { } // O_DIRECTORY: reject non-directories - if ((flags & O_DIRECTORY) && (mode & S_IFMT) !== S_IFDIR) { + if (flags & O_DIRECTORY && (mode & S_IFMT) !== S_IFDIR) { throw new SFSError(ENOTDIR); } @@ -1288,31 +2355,33 @@ export class SharedFS { if (flags & O_TRUNC) { if ((mode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); this.inodeWriteLock(ino); - this.inodeTruncate(ino, 0); + this.inodeTruncate(ino, 0, true); this.inodeWriteUnlock(ino); } const fd = this.fdAlloc(ino, flags, false); if (fd < 0) throw new SFSError(fd); - // If append, set offset to end - if (flags & O_APPEND) { - const base = FD_TABLE_OFFSET + fd * FD_ENTRY_SIZE; - this.w64(base + FD_OFFSET, this.r64(inoOff + INO_SIZE)); - } - return fd; } close(fd: number): void { + this.withNamespaceLock(() => this.closeUnlocked(fd)); + } + + private closeUnlocked(fd: number): void { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); this.fdFree(fd); + this.inodeDropOpenRef(entry.ino); } read(fd: number, buffer: Uint8Array): number { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); + const inoOff = this.inodeOffset(entry.ino); + const mode = this.r32(inoOff + INO_MODE); + if ((mode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); this.inodeReadLock(entry.ino); try { @@ -1345,6 +2414,12 @@ export class SharedFS { const inoOff = this.inodeOffset(entry.ino); offset = this.r64(inoOff + INO_SIZE); } + if (!Number.isSafeInteger(offset) || offset < 0) { + throw new SFSError(EINVAL); + } + if (offset > MAX_FILE_SIZE || data.length > MAX_FILE_SIZE - offset) { + throw new SFSError(EFBIG); + } const nwritten = this.inodeWriteData( entry.ino, @@ -1352,6 +2427,7 @@ export class SharedFS { data, data.length, ); + if (nwritten < 0) return nwritten; // Update offset const base = FD_TABLE_OFFSET + fd * FD_ENTRY_SIZE; this.w64(base + FD_OFFSET, offset + nwritten); @@ -1378,7 +2454,7 @@ export class SharedFS { throw new SFSError(EINVAL); } - if (newOffset < 0) throw new SFSError(EINVAL); + this.validateFileSize(newOffset); const base = FD_TABLE_OFFSET + fd * FD_ENTRY_SIZE; this.w64(base + FD_OFFSET, newOffset); @@ -1389,10 +2465,11 @@ export class SharedFS { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); if ((entry.flags & O_ACCMODE) === O_RDONLY) throw new SFSError(EBADF); + this.validateFileSize(length); this.inodeWriteLock(entry.ino); try { - this.inodeTruncate(entry.ino, length); + this.inodeTruncate(entry.ino, length, true); } finally { this.inodeWriteUnlock(entry.ino); } @@ -1412,6 +2489,10 @@ export class SharedFS { // ── Public API: Path operations ────────────────────────────────── stat(path: string): StatResult { + return this.withNamespaceLock(() => this.statUnlocked(path)); + } + + private statUnlocked(path: string): StatResult { const ino = this.pathResolve(path, true); if (ino < 0) throw new SFSError(ino); this.inodeReadLock(ino); @@ -1423,6 +2504,10 @@ export class SharedFS { } lstat(path: string): StatResult { + return this.withNamespaceLock(() => this.lstatUnlocked(path)); + } + + private lstatUnlocked(path: string): StatResult { const ino = this.pathResolve(path, false); // don't follow symlinks if (ino < 0) throw new SFSError(ino); this.inodeReadLock(ino); @@ -1433,9 +2518,14 @@ export class SharedFS { } } - unlink(path: string): void { + unlink(path: string): NamespaceEntryIdentity { + return this.withNamespaceLock(() => this.unlinkUnlocked(path)); + } + + private unlinkUnlocked(path: string): NamespaceEntryIdentity { const { parentIno, name } = this.pathResolveParent(path); const nameBytes = encoder.encode(name); + const requiresDirectory = path.length > 1 && path.endsWith("/"); this.inodeWriteLock(parentIno); try { @@ -1444,34 +2534,48 @@ export class SharedFS { const childOff = this.inodeOffset(childIno); const mode = this.r32(childOff + INO_MODE); + if (requiresDirectory && (mode & S_IFMT) !== S_IFDIR) { + throw new SFSError(ENOTDIR); + } if ((mode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); + const removed = this.namespaceEntryIdentity(childIno); const rc = this.dirRemoveEntry(parentIno, nameBytes); if (rc < 0) throw new SFSError(rc); + let shouldFree = false; this.inodeWriteLock(childIno); - const linkCount = this.r32(childOff + INO_LINK_COUNT); - if (linkCount <= 1) { - this.inodeTruncate(childIno, 0); - this.w32(childOff + INO_LINK_COUNT, 0); - this.inodeWriteUnlock(childIno); - this.inodeFree(childIno); - } else { - this.w32(childOff + INO_LINK_COUNT, linkCount - 1); + try { + shouldFree = this.inodeDropLinkRefLocked(childIno); + } finally { this.inodeWriteUnlock(childIno); } + if (shouldFree) this.inodeFree(childIno); + return removed; } finally { this.inodeWriteUnlock(parentIno); } } - rename(oldPath: string, newPath: string): void { + rename(oldPath: string, newPath: string): RenameIdentityResult { + return this.withNamespaceLock(() => this.renameUnlocked(oldPath, newPath)); + } + + private renameUnlocked( + oldPath: string, + newPath: string, + ): RenameIdentityResult { const { parentIno: oldParent, name: oldName } = this.pathResolveParent(oldPath); const { parentIno: newParent, name: newName } = this.pathResolveParent(newPath); + if (isReservedDirectoryName(oldName) || isReservedDirectoryName(newName)) { + throw new SFSError(EINVAL); + } const oldNameBytes = encoder.encode(oldName); const newNameBytes = encoder.encode(newName); + const oldRequiresDirectory = oldPath.length > 1 && oldPath.endsWith("/"); + const newRequiresDirectory = newPath.length > 1 && newPath.endsWith("/"); // Lock both parents (consistent order to avoid deadlock) const first = Math.min(oldParent, newParent); @@ -1482,46 +2586,119 @@ export class SharedFS { try { const srcIno = this.dirLookup(oldParent, oldNameBytes); if (srcIno < 0) throw new SFSError(srcIno); + const srcOff = this.inodeOffset(srcIno); + const srcMode = this.r32(srcOff + INO_MODE); + const srcType = srcMode & S_IFMT; + const source = this.namespaceEntryIdentity(srcIno); - // Remove any existing entry at destination + if ( + (oldRequiresDirectory || newRequiresDirectory) && + srcType !== S_IFDIR + ) { + throw new SFSError(ENOTDIR); + } + + if (srcType === S_IFDIR && this.dirIsAncestor(srcIno, newParent)) { + throw new SFSError(EINVAL); + } + + // Replace an existing destination entry in place. Removing it before + // allocating/inserting the source can destroy the destination when the + // rename later fails with ENOSPC. const existingIno = this.dirLookup(newParent, newNameBytes); + let removedExistingDirectory = false; + let replaced: NamespaceEntryIdentity | undefined; if (existingIno >= 0) { + if (existingIno === srcIno) { + return { source, replaced: source }; + } + replaced = this.namespaceEntryIdentity(existingIno); const existOff = this.inodeOffset(existingIno); const existMode = this.r32(existOff + INO_MODE); - if ((existMode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); - this.dirRemoveEntry(newParent, newNameBytes); - this.inodeWriteLock(existingIno); - this.inodeTruncate(existingIno, 0); - this.w32(existOff + INO_LINK_COUNT, 0); - this.inodeWriteUnlock(existingIno); - this.inodeFree(existingIno); - } + const existType = existMode & S_IFMT; - // Add entry in new directory - const rc = this.dirAddEntry(newParent, newNameBytes, srcIno); - if (rc < 0) throw new SFSError(rc); + if (srcType === S_IFDIR && existType !== S_IFDIR) { + throw new SFSError(ENOTDIR); + } + if (srcType !== S_IFDIR && existType === S_IFDIR) { + throw new SFSError(EISDIR); + } + + let shouldFreeExisting = false; + const existingAlreadyLocked = + existingIno === oldParent || existingIno === newParent; + if (!existingAlreadyLocked) this.inodeWriteLock(existingIno); + try { + if (existType === S_IFDIR && !this.dirIsEmpty(existingIno)) { + throw new SFSError(ENOTEMPTY); + } + const replaceRc = this.dirReplaceEntryIno( + newParent, + newNameBytes, + srcIno, + ); + if (replaceRc < 0) throw new SFSError(replaceRc); + shouldFreeExisting = + existType === S_IFDIR + ? this.inodeOrphanLocked(existingIno) + : this.inodeDropLinkRefLocked(existingIno); + } finally { + if (!existingAlreadyLocked) this.inodeWriteUnlock(existingIno); + } + if (shouldFreeExisting) this.inodeFree(existingIno); + removedExistingDirectory = existType === S_IFDIR; + } else { + const addRc = this.dirAddEntry(newParent, newNameBytes, srcIno); + if (addRc < 0) throw new SFSError(addRc); + } // Remove entry from old directory - this.dirRemoveEntry(oldParent, oldNameBytes); + const removeRc = this.dirRemoveEntry(oldParent, oldNameBytes); + if (removeRc < 0) throw new SFSError(removeRc); // Update link counts for directory renames - const srcOff = this.inodeOffset(srcIno); - const srcMode = this.r32(srcOff + INO_MODE); - if ( - (srcMode & S_IFMT) === S_IFDIR && - oldParent !== newParent - ) { - const oldPOff = this.inodeOffset(oldParent); - this.w32( - oldPOff + INO_LINK_COUNT, - this.r32(oldPOff + INO_LINK_COUNT) - 1, - ); + if (srcType === S_IFDIR) { + if (oldParent !== newParent) { + const oldPOff = this.inodeOffset(oldParent); + this.w32( + oldPOff + INO_LINK_COUNT, + this.r32(oldPOff + INO_LINK_COUNT) - 1, + ); + const newPOff = this.inodeOffset(newParent); + this.w32( + newPOff + INO_LINK_COUNT, + this.r32(newPOff + INO_LINK_COUNT) + 1, + ); + + this.inodeWriteLock(srcIno); + try { + const dotdotRc = this.dirReplaceEntryIno( + srcIno, + DOTDOT_BYTES, + newParent, + ); + if (dotdotRc < 0) throw new SFSError(dotdotRc); + this.w64(srcOff + INO_CTIME, Date.now()); + } finally { + this.inodeWriteUnlock(srcIno); + } + } + + if (removedExistingDirectory) { + const newPOff = this.inodeOffset(newParent); + this.w32( + newPOff + INO_LINK_COUNT, + this.r32(newPOff + INO_LINK_COUNT) - 1, + ); + } + } else if (removedExistingDirectory) { const newPOff = this.inodeOffset(newParent); this.w32( newPOff + INO_LINK_COUNT, - this.r32(newPOff + INO_LINK_COUNT) + 1, + this.r32(newPOff + INO_LINK_COUNT) - 1, ); } + return { source, replaced }; } finally { if (first !== second) this.inodeWriteUnlock(second); this.inodeWriteUnlock(first); @@ -1529,6 +2706,10 @@ export class SharedFS { } mkdir(path: string, mode: number = 0o755): void { + this.withNamespaceLock(() => this.mkdirUnlocked(path, mode)); + } + + private mkdirUnlocked(path: string, mode: number = 0o755): void { const { parentIno, name } = this.pathResolveParent(path); const nameBytes = encoder.encode(name); @@ -1594,7 +2775,12 @@ export class SharedFS { } rmdir(path: string): void { + this.withNamespaceLock(() => this.rmdirUnlocked(path)); + } + + private rmdirUnlocked(path: string): void { const { parentIno, name } = this.pathResolveParent(path); + if (isReservedDirectoryName(name)) throw new SFSError(EINVAL); const nameBytes = encoder.encode(name); this.inodeWriteLock(parentIno); @@ -1606,17 +2792,18 @@ export class SharedFS { const mode = this.r32(childOff + INO_MODE); if ((mode & S_IFMT) !== S_IFDIR) throw new SFSError(ENOTDIR); + let shouldFree = false; this.inodeWriteLock(childIno); try { if (!this.dirIsEmpty(childIno)) throw new SFSError(ENOTEMPTY); - this.dirRemoveEntry(parentIno, nameBytes); - this.inodeTruncate(childIno, 0); - this.w32(childOff + INO_LINK_COUNT, 0); + const removeRc = this.dirRemoveEntry(parentIno, nameBytes); + if (removeRc < 0) throw new SFSError(removeRc); + shouldFree = this.inodeOrphanLocked(childIno); } finally { this.inodeWriteUnlock(childIno); } - this.inodeFree(childIno); + if (shouldFree) this.inodeFree(childIno); // Decrement parent link count const pOff = this.inodeOffset(parentIno); @@ -1627,6 +2814,10 @@ export class SharedFS { } symlink(target: string, linkPath: string): void { + this.withNamespaceLock(() => this.symlinkUnlocked(target, linkPath)); + } + + private symlinkUnlocked(target: string, linkPath: string): void { const { parentIno, name } = this.pathResolveParent(linkPath); const nameBytes = encoder.encode(name); const targetBytes = encoder.encode(target); @@ -1655,14 +2846,25 @@ export class SharedFS { targetBytes, targetBytes.length, ); - if (written < 0) { + if (written !== targetBytes.length) { + if (written > 0) this.inodeTruncate(newIno, 0); this.inodeFree(newIno); - throw new SFSError(written); + throw new SFSError(written < 0 ? written : ENOSPC); } } const rc = this.dirAddEntry(parentIno, nameBytes, newIno); if (rc < 0) { + if (targetBytes.length <= INLINE_SYMLINK_SIZE) { + this.u8.fill( + 0, + newOff + INO_DIRECT, + newOff + INO_DIRECT + INLINE_SYMLINK_SIZE, + ); + this.w64(newOff + INO_SIZE, 0); + } else { + this.inodeTruncate(newIno, 0); + } this.inodeFree(newIno); throw new SFSError(rc); } @@ -1672,6 +2874,10 @@ export class SharedFS { } chmod(path: string, mode: number): void { + this.withNamespaceLock(() => this.chmodUnlocked(path, mode)); + } + + private chmodUnlocked(path: string, mode: number): void { const ino = this.pathResolve(path, true); if (ino < 0) throw new SFSError(ino); this.inodeWriteLock(ino); @@ -1700,6 +2906,10 @@ export class SharedFS { } chown(path: string, uid: number, gid: number): void { + this.withNamespaceLock(() => this.chownUnlocked(path, uid, gid)); + } + + private chownUnlocked(path: string, uid: number, gid: number): void { const ino = this.pathResolve(path, true); // POSIX chown follows symlinks if (ino < 0) throw new SFSError(ino); this.inodeWriteLock(ino); @@ -1730,6 +2940,10 @@ export class SharedFS { } lchown(path: string, uid: number, gid: number): void { + this.withNamespaceLock(() => this.lchownUnlocked(path, uid, gid)); + } + + private lchownUnlocked(path: string, uid: number, gid: number): void { const ino = this.pathResolve(path, false); // no-follow: chowns the symlink itself if (ino < 0) throw new SFSError(ino); this.inodeWriteLock(ino); @@ -1743,7 +2957,25 @@ export class SharedFS { } } - utimens(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { + utimens( + path: string, + atimeSec: number, + atimeNsec: number, + mtimeSec: number, + mtimeNsec: number, + ): void { + this.withNamespaceLock(() => + this.utimensUnlocked(path, atimeSec, atimeNsec, mtimeSec, mtimeNsec), + ); + } + + private utimensUnlocked( + path: string, + atimeSec: number, + atimeNsec: number, + mtimeSec: number, + mtimeNsec: number, + ): void { const ino = this.pathResolve(path, true); if (ino < 0) throw new SFSError(ino); this.inodeWriteLock(ino); @@ -1753,11 +2985,17 @@ export class SharedFS { const UTIME_OMIT = 0x3ffffffe; const now = Date.now(); if (atimeNsec !== UTIME_OMIT) { - const atimeMs = atimeNsec === UTIME_NOW ? now : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); + const atimeMs = + atimeNsec === UTIME_NOW + ? now + : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); this.w64(off + INO_ATIME, atimeMs); } if (mtimeNsec !== UTIME_OMIT) { - const mtimeMs = mtimeNsec === UTIME_NOW ? now : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); + const mtimeMs = + mtimeNsec === UTIME_NOW + ? now + : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); this.w64(off + INO_MTIME, mtimeMs); } this.w64(off + INO_CTIME, now); @@ -1766,8 +3004,19 @@ export class SharedFS { } } - link(existingPath: string, newPath: string): void { - const srcIno = this.pathResolve(existingPath, true); + link(existingPath: string, newPath: string): NamespaceEntryIdentity { + return this.withNamespaceLock(() => + this.linkUnlocked(existingPath, newPath), + ); + } + + private linkUnlocked( + existingPath: string, + newPath: string, + ): NamespaceEntryIdentity { + // Select POSIX link(2)'s permitted no-follow behavior for a final + // symlink, which also matches linkat() when AT_SYMLINK_FOLLOW is clear. + const srcIno = this.pathResolve(existingPath, false); if (srcIno < 0) throw new SFSError(srcIno); const srcOff = this.inodeOffset(srcIno); const srcMode = this.r32(srcOff + INO_MODE); @@ -1790,12 +3039,20 @@ export class SharedFS { } finally { this.inodeWriteUnlock(srcIno); } + return { + ...this.namespaceEntryIdentity(srcIno), + linkCount: this.r32(srcOff + INO_LINK_COUNT), + }; } finally { this.inodeWriteUnlock(parentIno); } } readlink(path: string): string { + return this.withNamespaceLock(() => this.readlinkUnlocked(path)); + } + + private readlinkUnlocked(path: string): string { const ino = this.pathResolve(path, false); if (ino < 0) throw new SFSError(ino); @@ -1823,6 +3080,10 @@ export class SharedFS { // ── Public API: Directory reading ──────────────────────────────── opendir(path: string): number { + return this.withNamespaceLock(() => this.opendirUnlocked(path)); + } + + private opendirUnlocked(path: string): number { const ino = this.pathResolve(path, true); if (ino < 0) throw new SFSError(ino); @@ -1835,7 +3096,11 @@ export class SharedFS { return dd; } - readdirEntry( + readdirEntry(dd: number): { name: string; stat: StatResult } | null { + return this.withNamespaceLock(() => this.readdirEntryUnlocked(dd)); + } + + private readdirEntryUnlocked( dd: number, ): { name: string; stat: StatResult } | null { const entry = this.fdGet(dd); @@ -1849,14 +3114,22 @@ export class SharedFS { const fileBlock = Math.floor(pos / BLOCK_SIZE); const blockOff = pos % BLOCK_SIZE; const phys = this.inodeBlockMap(entry.ino, fileBlock, false); - if (phys <= 0) return null; + if (phys <= 0) throw new SFSError(EIO); const abs = phys * BLOCK_SIZE + blockOff; const entIno = this.r32(abs); const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return null; + if ( + !this.isValidDirEntry( + blockOff, + Math.min(BLOCK_SIZE, blockOff + dirSize - pos), + recLen, + entNameLen, + ) + ) + throw new SFSError(EIO); // Advance offset — update both the SAB (persistent) and the local // snapshot so the while loop progresses past deleted entries (entIno=0). @@ -1865,6 +3138,10 @@ export class SharedFS { this.w64(base + FD_OFFSET, pos + recLen); if (entIno !== 0) { + if (entIno >= this.r32(SB_TOTAL_INODES)) throw new SFSError(EIO); + const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; + const word = this.r32(ibStart + (entIno >> 5) * 4); + if ((word & (1 << (entIno & 31))) === 0) throw new SFSError(EIO); const nameStr = safeDecode( this.u8.subarray( abs + DIRENT_HEADER_SIZE, diff --git a/host/src/vfs/time.ts b/host/src/vfs/time.ts index 0136cd676..4a9f82c6d 100644 --- a/host/src/vfs/time.ts +++ b/host/src/vfs/time.ts @@ -23,8 +23,8 @@ export class NodeTimeProvider implements TimeProvider { const elapsed = ns - this._startNs; return { sec: Number(elapsed / 1000000000n), nsec: Number(elapsed % 1000000000n) }; } - if (clockId === 1) { - // CLOCK_MONOTONIC + if (clockId === 1 || clockId === 7) { + // CLOCK_MONOTONIC / CLOCK_BOOTTIME return { sec: Number(ns / 1000000000n), nsec: Number(ns % 1000000000n) }; } // CLOCK_REALTIME — use hrtime + epoch offset for nanosecond resolution @@ -43,8 +43,8 @@ export class NodeTimeProvider implements TimeProvider { export class BrowserTimeProvider implements TimeProvider { clockGettime(clockId: number): { sec: number; nsec: number } { - if (clockId === 1 || clockId === 2 || clockId === 3) { - // CLOCK_MONOTONIC / CLOCK_PROCESS_CPUTIME_ID / CLOCK_THREAD_CPUTIME_ID + if (clockId === 1 || clockId === 2 || clockId === 3 || clockId === 7) { + // CLOCK_MONOTONIC / CPU-time clocks / CLOCK_BOOTTIME const ms = performance.now(); return { sec: Math.floor(ms / 1000), nsec: Math.floor((ms % 1000) * 1_000_000) }; } diff --git a/host/src/vm-interrupt-timer.ts b/host/src/vm-interrupt-timer.ts new file mode 100644 index 000000000..988609715 --- /dev/null +++ b/host/src/vm-interrupt-timer.ts @@ -0,0 +1,230 @@ +/** + * A process-scoped cooperative VM-interrupt timer. + * + * Process workers cannot reliably run their own timer while executing a + * CPU-bound Wasm loop. The kernel worker owns this timer instead and writes + * the runtime's interrupt flags through the process's shared memory. + * + * The generation object is deliberately part of every entry. A numeric PID + * can be reused by exec or a later process, so neither a queued timer callback + * nor a stale worker message may act on whatever generation happens to own the + * PID later. + */ + +export const MAX_VM_INTERRUPT_TIMER_DELAY_MS = 0x7fffffff; + +export interface VmInterruptProcessGeneration { + readonly memory: WebAssembly.Memory; +} + +export interface VmInterruptTimerRequest { + timedOutPtr: number; + vmInterruptPtr: number; + seconds: number; +} + +export interface VmInterruptTimerScheduler> { + /** Monotonic milliseconds. */ + now(): number; + set(callback: () => void, delayMs: number): Handle; + clear(handle: Handle): void; +} + +interface TimerEntry { + generation: Generation; + deadlineMs: number; + timedOutPtr: number; + vmInterruptPtr: number; + handle?: Handle; +} + +function defaultScheduler(): VmInterruptTimerScheduler { + return { + now: () => performance.now(), + set: (callback, delayMs) => setTimeout(callback, delayMs), + clear: (handle) => clearTimeout(handle), + }; +} + +function sharedFlags( + generation: VmInterruptProcessGeneration, + timedOutPtr: number, + vmInterruptPtr: number, +): Uint8Array | null { + // TypeScript's WebAssembly.Memory.buffer declaration is ArrayBuffer even + // when the memory descriptor used `shared: true`; narrow from unknown so a + // real SharedArrayBuffer remains representable without an impossible + // ArrayBuffer & SharedArrayBuffer intersection. + const buffer: unknown = generation.memory.buffer; + if ( + typeof SharedArrayBuffer === "undefined" || + !(buffer instanceof SharedArrayBuffer) + ) { + return null; + } + if ( + !Number.isSafeInteger(timedOutPtr) || + timedOutPtr < 0 || + timedOutPtr >= buffer.byteLength || + !Number.isSafeInteger(vmInterruptPtr) || + vmInterruptPtr < 0 || + vmInterruptPtr >= buffer.byteLength + ) { + return null; + } + return new Uint8Array(buffer); +} + +export class VmInterruptTimerManager< + Generation extends VmInterruptProcessGeneration, + Handle = ReturnType, +> { + private readonly entries = new Map>(); + + constructor( + private readonly currentGeneration: (pid: number) => Generation | undefined, + private readonly scheduler: VmInterruptTimerScheduler = + defaultScheduler() as VmInterruptTimerScheduler, + ) {} + + /** + * Apply the runtime hook's arm/cancel request for the listener-owned PID. + * Non-positive durations cancel the current generation's timer. + */ + handleRequest( + pid: number, + generation: Generation, + request: VmInterruptTimerRequest, + ): boolean { + if (request.seconds > 0) { + return this.arm(pid, generation, request); + } + return this.cancel(pid, generation); + } + + /** Replace any existing timer for the same current process generation. */ + arm( + pid: number, + generation: Generation, + request: VmInterruptTimerRequest, + ): boolean { + if (this.currentGeneration(pid) !== generation) return false; + + // A new request replaces the previous timer even when the new request is + // malformed. This matches timer-set semantics and prevents an old deadline + // from surviving a rejected re-arm. + this.clear(pid); + + if (!Number.isFinite(request.seconds) || !(request.seconds > 0)) return false; + if (!sharedFlags(generation, request.timedOutPtr, request.vmInterruptPtr)) { + return false; + } + + const now = this.scheduler.now(); + const delayMs = request.seconds * 1000; + const deadlineMs = now + delayMs; + if (!Number.isFinite(now) || !Number.isFinite(delayMs) || !Number.isFinite(deadlineMs)) { + return false; + } + + const entry: TimerEntry = { + generation, + deadlineMs, + timedOutPtr: request.timedOutPtr, + vmInterruptPtr: request.vmInterruptPtr, + }; + this.entries.set(pid, entry); + this.schedule(pid, entry); + return true; + } + + /** Cancel only when the caller still owns the current PID generation. */ + cancel(pid: number, generation: Generation): boolean { + if (this.currentGeneration(pid) !== generation) return false; + this.clear(pid, generation); + return true; + } + + /** Clear a PID timer, optionally restricted to one exact generation. */ + clear(pid: number, generation?: Generation): boolean { + const entry = this.entries.get(pid); + if (!entry || (generation !== undefined && entry.generation !== generation)) { + return false; + } + if (entry.handle !== undefined) { + this.scheduler.clear(entry.handle); + entry.handle = undefined; + } + this.entries.delete(pid); + return true; + } + + clearAll(): void { + for (const [pid] of this.entries) this.clear(pid); + } + + get activeCount(): number { + return this.entries.size; + } + + private schedule(pid: number, entry: TimerEntry): void { + if ( + this.entries.get(pid) !== entry || + this.currentGeneration(pid) !== entry.generation + ) { + this.discardIfCurrent(pid, entry); + return; + } + + const remainingMs = entry.deadlineMs - this.scheduler.now(); + if (remainingMs <= 0) { + this.fire(pid, entry); + return; + } + + // Browser and Node timers clamp/overflow beyond a signed 32-bit delay. + // Schedule in chunks and recompute against the monotonic deadline after + // every wake. Ceil prevents a fractional delay from firing early. + const delayMs = Math.min( + MAX_VM_INTERRUPT_TIMER_DELAY_MS, + Math.max(1, Math.ceil(remainingMs)), + ); + const handle = this.scheduler.set(() => { + if (this.entries.get(pid) !== entry || entry.handle !== handle) return; + entry.handle = undefined; + this.schedule(pid, entry); + }, delayMs); + entry.handle = handle; + } + + private fire(pid: number, entry: TimerEntry): void { + if ( + this.entries.get(pid) !== entry || + this.currentGeneration(pid) !== entry.generation + ) { + this.discardIfCurrent(pid, entry); + return; + } + + const flags = sharedFlags( + entry.generation, + entry.timedOutPtr, + entry.vmInterruptPtr, + ); + this.entries.delete(pid); + entry.handle = undefined; + if (!flags) return; + + Atomics.store(flags, entry.timedOutPtr, 1); + Atomics.store(flags, entry.vmInterruptPtr, 1); + } + + private discardIfCurrent(pid: number, entry: TimerEntry): void { + if (this.entries.get(pid) !== entry) return; + if (entry.handle !== undefined) { + this.scheduler.clear(entry.handle); + entry.handle = undefined; + } + this.entries.delete(pid); + } +} diff --git a/host/src/worker-adapter.ts b/host/src/worker-adapter.ts index 19c919f48..0f9045a0e 100644 --- a/host/src/worker-adapter.ts +++ b/host/src/worker-adapter.ts @@ -97,16 +97,49 @@ export class MockWorkerAdapter implements WorkerAdapter { // --- Node.js implementation --- -import { Worker } from "node:worker_threads"; +import { Worker, type WorkerOptions } from "node:worker_threads"; import { pathToFileURL } from "node:url"; import { createRequire } from "node:module"; import { existsSync } from "node:fs"; +// Wasm guest stacks consume the embedding worker's native stack when engines +// recurse through Wasm frames. Keep the default high enough for stack-heavy +// POSIX workloads while retaining an environment override for constrained +// embedders. +const DEFAULT_NODE_WORKER_STACK_SIZE_MB = 32; + function currentModuleUrl(): string { if (typeof __filename !== "undefined") return pathToFileURL(__filename).href; return import.meta.url; } +/** @internal Exported so the host policy can be validated without spawning a worker. */ +export function nodeWorkerStackSizeMb( + raw = process.env.KANDELO_NODE_WORKER_STACK_SIZE_MB, +): number { + if (raw === undefined || raw === "") return DEFAULT_NODE_WORKER_STACK_SIZE_MB; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`invalid KANDELO_NODE_WORKER_STACK_SIZE_MB: ${raw}`); + } + return parsed; +} + +/** @internal Exported so tests can verify resource-limit composition. */ +export function nodeWorkerOptions( + workerData: unknown, + options: WorkerOptions = {}, +): WorkerOptions { + return { + ...options, + workerData, + resourceLimits: { + ...options.resourceLimits, + stackSizeMb: nodeWorkerStackSizeMb(), + }, + }; +} + export class NodeWorkerAdapter implements WorkerAdapter { private entryUrl: URL; private _compiledEntry: URL | false | undefined; @@ -153,7 +186,7 @@ export class NodeWorkerAdapter implements WorkerAdapter { // bootstrap which takes >500ms with 10+ concurrent workers). const compiledEntry = this.resolveCompiledEntry(); if (compiledEntry) { - const worker = new Worker(compiledEntry, { workerData }); + const worker = new Worker(compiledEntry, nodeWorkerOptions(workerData)); return new NodeWorkerHandle(worker); } @@ -169,10 +202,9 @@ export class NodeWorkerAdapter implements WorkerAdapter { `await import('${entryUrl}');`, ].join("\n"); - const worker = new Worker(bootstrap, { + const worker = new Worker(bootstrap, nodeWorkerOptions(workerData, { eval: true, - workerData, - }); + })); return new NodeWorkerHandle(worker); } } diff --git a/host/src/worker-main.ts b/host/src/worker-main.ts index 923443c49..f9b40f73c 100644 --- a/host/src/worker-main.ts +++ b/host/src/worker-main.ts @@ -11,10 +11,12 @@ import type { WorkerToHostMessage, } from "./worker-protocol"; import { + createLongjmpTag, DynamicLinker, FORK_CAP_DYLINK_MAIN, forkInstrumentRoleAvailable, readForkInstrumentCapabilityClaim, + requireLongjmpTag, type LoadedSharedLibrary, type SideModuleForkState, } from "./dylink"; @@ -213,6 +215,7 @@ function buildDlopenImports( getStackPointer: () => WebAssembly.Global | undefined, getInstance: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8, + longjmpTag: WebAssembly.Tag | undefined, mainHasDylinkForkRole: boolean, ): DlopenSupport { let linker: DynamicLinker | null = null; @@ -312,7 +315,7 @@ function buildDlopenImports( // not be shadowed by main exports. const RESERVED = new Set([ "memory", "__indirect_function_table", - "__memory_base", "__table_base", "__stack_pointer", + "__memory_base", "__table_base", "__stack_pointer", "__c_longjmp", ]); const globalSymbols = new Map(); const inst = getInstance(); @@ -380,6 +383,8 @@ function buildDlopenImports( globalSymbols, got: new Map(), loadedLibraries, + longjmpTag, + ptrWidth, mainModuleSymbols, sideModuleFork, sideModuleForkUnavailableReason: !mainHasDylinkForkRole @@ -668,6 +673,12 @@ function buildImportObject( dlopenImports?: Record, getInstance?: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8 = 4, + longjmpTag?: WebAssembly.Tag, + postVmInterruptTimer?: ( + timedOutPtr: number, + vmInterruptPtr: number, + seconds: number, + ) => void, ): WebAssembly.Imports { const envImports: Record = { memory }; /** Convert wasm64 BigInt pointer to number (safe since addresses < 4GB) */ @@ -687,14 +698,14 @@ function buildImportObject( } } - // llvm/lld ≥22 emit __c_longjmp as a tag import for setjmp users; instantiation fails silently without it. + // LLVM/lld >= 22 import this tag for setjmp users. The process owns its + // identity so a longjmp thrown through a side module can be caught by the + // main image (and vice versa). if (moduleImports.some(i => i.module === "env" && i.name === "__c_longjmp" && (i.kind as string) === "tag")) { - const Tag = (WebAssembly as typeof WebAssembly & { - Tag?: new (descriptor: { parameters: string[] }) => WebAssembly.Tag; - }).Tag; - if (Tag) { - envImports.__c_longjmp = new Tag({ parameters: ["i32"] }) as unknown as WebAssembly.ExportValue; - } + envImports.__c_longjmp = requireLongjmpTag( + longjmpTag, + "process module", + ) as unknown as WebAssembly.ExportValue; } // Add dlopen imports if provided @@ -702,6 +713,26 @@ function buildImportObject( Object.assign(envImports, dlopenImports); } + if ( + moduleImports.some( + (i) => + i.module === "env" && + i.name === "__wasm_posix_vm_interrupt_after" && + i.kind === "function", + ) + ) { + if (!postVmInterruptTimer) { + throw new Error("VM interrupt timer import requested without a host timer route"); + } + envImports.__wasm_posix_vm_interrupt_after = ( + timedOutPtr: number | bigint, + vmInterruptPtr: number | bigint, + seconds: number | bigint, + ): void => { + postVmInterruptTimer(n(timedOutPtr), n(vmInterruptPtr), n(seconds)); + }; + } + // C++ operator new/delete fallbacks — delegate to the wasm instance's malloc/free. // Normally resolved by MariaDB's my_new.cc (USE_MYSYS_NEW), but kept as safety net. if (getInstance) { @@ -1084,6 +1115,7 @@ export async function centralizedWorkerMain( } // --- SDK module path (existing) --- + const processLongjmpTag = createLongjmpTag(ptrWidth); let kernelExitStatus: number | null = null; const kernelImports = buildKernelImports( memory, @@ -1139,10 +1171,20 @@ export async function centralizedWorkerMain( () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, + processLongjmpTag, hasDylinkForkRole, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth); + () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + (timedOutPtr, vmInterruptPtr, seconds) => { + port.postMessage({ + type: "vm_interrupt_timer", + pid, + timedOutPtr, + vmInterruptPtr, + seconds, + } satisfies WorkerToHostMessage); + }); const instance = await WebAssembly.instantiate(module, importObject); processInstance = instance; verifyProgramAbi(programBytes, initData.kernelAbiVersion, pid); @@ -1294,10 +1336,20 @@ export async function centralizedWorkerMain( () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, + processLongjmpTag, false, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth); + () => processInstance ?? undefined, ptrWidth, processLongjmpTag, + (timedOutPtr, vmInterruptPtr, seconds) => { + port.postMessage({ + type: "vm_interrupt_timer", + pid, + timedOutPtr, + vmInterruptPtr, + seconds, + } satisfies WorkerToHostMessage); + }); const instance = await WebAssembly.instantiate(module, importObject); processInstance = instance; verifyProgramAbi(programBytes, initData.kernelAbiVersion, pid); @@ -1992,8 +2044,18 @@ export async function centralizedThreadWorkerMain( ); }; } + const threadLongjmpTag = createLongjmpTag(ptrWidth); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, undefined, - () => threadInstance, ptrWidth); + () => threadInstance, ptrWidth, threadLongjmpTag, + (timedOutPtr, vmInterruptPtr, seconds) => { + port.postMessage({ + type: "vm_interrupt_timer", + pid, + timedOutPtr, + vmInterruptPtr, + seconds, + } satisfies WorkerToHostMessage); + }); const instance = new WebAssembly.Instance(module, importObject); threadInstance = instance; diff --git a/host/src/worker-protocol.ts b/host/src/worker-protocol.ts index b7c4b127c..c837a91ab 100644 --- a/host/src/worker-protocol.ts +++ b/host/src/worker-protocol.ts @@ -99,7 +99,8 @@ export type WorkerToHostMessage = | WorkerErrorMessage | ExecRequestMessage | ExecCompleteMessage - | AlarmSetMessage; + | AlarmSetMessage + | VmInterruptTimerMessage; export interface WorkerReadyMessage { type: "ready"; @@ -141,6 +142,14 @@ export interface AlarmSetMessage { seconds: number; } +export interface VmInterruptTimerMessage { + type: "vm_interrupt_timer"; + pid: number; + timedOutPtr: number; + vmInterruptPtr: number; + seconds: number; +} + export interface ExecReplyMessage { type: "exec_reply"; wasmBytes: ArrayBuffer; diff --git a/host/test/browser-kernel.test.ts b/host/test/browser-kernel.test.ts index 7107bb115..a1fc0d3de 100644 --- a/host/test/browser-kernel.test.ts +++ b/host/test/browser-kernel.test.ts @@ -155,6 +155,93 @@ describe("BrowserKernel", () => { expect(await exit).toBe(7); }); + it("delivers host diagnostics without contaminating guest stderr", async () => { + const BrowserKernel = await loadBrowserKernel(); + const onHostDiagnostic = vi.fn(); + const onStderr = vi.fn(); + const kernel = new BrowserKernel({ + kernelOwnedFs: true, + onHostDiagnostic, + onStderr, + }); + + const bootPromise = kernel.boot({ + kernelWasm: new ArrayBuffer(8), + vfsImage: new Uint8Array(0), + argv: ["/init"], + }); + await new Promise((r) => setTimeout(r, 0)); + const worker = MockWorker.instances[0]!; + worker.simulateMessage({ type: "ready" }); + await new Promise((r) => setTimeout(r, 0)); + const spawn = worker.lastMessage("spawn"); + worker.simulateMessage({ + type: "response", + requestId: spawn.requestId, + result: 100, + }); + await bootPromise; + + worker.simulateMessage({ + type: "host_diagnostic", + pid: 100, + status: 7, + source: "kernel process exit", + message: "[kernel-worker] nonzero process exit pid=100 status=7", + }); + + expect(onHostDiagnostic).toHaveBeenCalledOnce(); + expect(onHostDiagnostic).toHaveBeenCalledWith({ + pid: 100, + status: 7, + source: "kernel process exit", + message: "[kernel-worker] nonzero process exit pid=100 status=7", + }); + expect(onStderr).not.toHaveBeenCalled(); + }); + + it("reports a worker-level error as a host diagnostic, not guest stderr", async () => { + const BrowserKernel = await loadBrowserKernel(); + const onHostDiagnostic = vi.fn(); + const onStderr = vi.fn(); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const kernel = new BrowserKernel({ + kernelOwnedFs: true, + onHostDiagnostic, + onStderr, + }); + + const bootPromise = kernel.boot({ + kernelWasm: new ArrayBuffer(8), + vfsImage: new Uint8Array(0), + argv: ["/init"], + }); + await new Promise((r) => setTimeout(r, 0)); + const worker = MockWorker.instances[0]!; + worker.simulateMessage({ type: "ready" }); + await new Promise((r) => setTimeout(r, 0)); + const spawn = worker.lastMessage("spawn"); + worker.simulateMessage({ + type: "response", + requestId: spawn.requestId, + result: 100, + }); + await bootPromise; + + worker.onerror?.({ message: "worker crashed" }); + + expect(onHostDiagnostic).toHaveBeenCalledOnce(); + expect(onHostDiagnostic).toHaveBeenCalledWith({ + pid: 0, + source: "kernel worker", + message: "[BrowserKernel] kernel worker error: worker crashed", + }); + expect(onStderr).not.toHaveBeenCalled(); + expect(consoleError).toHaveBeenCalledWith( + "[BrowserKernel] kernel worker error: worker crashed", + ); + }); + it("init() waits for lazy VFS registration to be acknowledged", async () => { const { MemoryFileSystem } = await import("../src/vfs/memory-fs"); const BrowserKernel = await loadBrowserKernel(); @@ -180,8 +267,15 @@ describe("BrowserKernel", () => { expect(lazy).toBeDefined(); expect(typeof lazy.requestId).toBe("number"); expect(lazy.entries).toMatchObject([ - { path: "/bin/lazy", url: "/assets/lazy.wasm", size: 123 }, + { + path: "/bin/lazy", + paths: ["/bin/lazy"], + url: "/assets/lazy.wasm", + size: 123, + }, ]); + expect(typeof lazy.entries[0].generation).toBe("number"); + expect(typeof lazy.entries[0].dataSequence).toBe("number"); expect(resolved).toBe(false); w.simulateMessage({ type: "response", requestId: lazy.requestId, result: true }); diff --git a/host/test/centralized-spawn.test.ts b/host/test/centralized-spawn.test.ts index ee078c5b9..7566b0ce9 100644 --- a/host/test/centralized-spawn.test.ts +++ b/host/test/centralized-spawn.test.ts @@ -100,4 +100,36 @@ describe("non-forking posix_spawn", () => { rmSync(tempDir, { recursive: true, force: true }); } }); + + it("reports ENOEXEC for malformed Wasm before creating the spawn child", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "kandelo-malformed-wasm-")); + const malformedWasm = join(tempDir, "malformed.wasm"); + try { + // Valid magic/version, followed by a truncated type section. A magic-only + // check accepts this; WebAssembly compilation must reject it in spawn's + // side-effect-free preflight, before the kernel applies file actions. + writeFileSync(malformedWasm, Buffer.from([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + 0x01, 0x01, 0xff, + ])); + + const result = await runCentralizedProgram({ + programPath: spawnSmokeWasm, + argv: ["spawn-smoke", "/usr/bin/malformed.wasm"], + execPrograms: new Map([ + ["/usr/bin/malformed.wasm", malformedWasm], + ]), + useDefaultRootfs: false, + timeout: 30_000, + }); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain("Exec format error"); + expect(result.stderr).not.toContain("Centralized worker failed"); + expect(result.stderr).not.toContain("WebAssembly.compile()"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); }); diff --git a/host/test/centralized-test-helper.ts b/host/test/centralized-test-helper.ts index 1241e68b9..cba55b7a1 100644 --- a/host/test/centralized-test-helper.ts +++ b/host/test/centralized-test-helper.ts @@ -21,6 +21,7 @@ import { type ProcessMemoryLayout, } from "../src/process-memory"; import { NodeKernelHost } from "../src/node-kernel-host"; +import type { HostDiagnostic } from "../src/host-diagnostic"; import type { CentralizedWorkerInitMessage, CentralizedThreadInitMessage, WorkerToHostMessage } from "../src/worker-protocol"; import type { PlatformIO } from "../src/types"; @@ -145,6 +146,8 @@ export interface RunProgramResult { exitCode: number; stdout: string; stderr: string; + /** Host-owned lifecycle/protocol diagnostics, never guest fd 2 bytes. */ + hostDiagnostics: HostDiagnostic[]; /** Raw stdout bytes (for binary output like compressed data) */ stdoutBytes: Uint8Array; /** Per-process fork counter for the spawned process, captured immediately @@ -179,6 +182,7 @@ async function runInWorkerThread(options: RunProgramOptions): Promise { stderr += new TextDecoder().decode(data); }, + onHostDiagnostic: (diagnostic) => { + hostDiagnostics.push(diagnostic); + }, }); await host.init(); @@ -270,7 +277,7 @@ async function runInWorkerThread(options: RunProgramOptions): Promise { + it("maps an invalid negative pid to ESRCH and rejects positive clock ID 10", async () => { + const result = await runCentralizedProgram({ + programPath: program, + argv: ["clock_getcpuclockid_test"], + useDefaultRootfs: false, + timeout: 10_000, + }); + + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stdout).toContain("PASS clock id validation"); + expect(result.stderr).toBe(""); + }); +}); diff --git a/host/test/dinit-image-helpers.test.ts b/host/test/dinit-image-helpers.test.ts new file mode 100644 index 000000000..4f92296d2 --- /dev/null +++ b/host/test/dinit-image-helpers.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { findRepoRoot } from "../src/binary-resolver"; +import { MemoryFileSystem } from "../src/vfs/memory-fs"; +import { addDinitBaseSystemFiles } from "../../images/vfs/scripts/dinit-image-helpers"; + +const O_RDONLY = 0; + +function readGuestFile(fs: MemoryFileSystem, path: string): string { + const size = fs.stat(path).size; + const fd = fs.open(path, O_RDONLY, 0); + try { + const bytes = new Uint8Array(size); + const count = fs.read(fd, bytes, null, bytes.byteLength); + return new TextDecoder().decode(bytes.subarray(0, count)); + } finally { + fs.close(fd); + } +} + +describe("dinit-derived image system databases", () => { + it("copies the authoritative rootfs services database without reducing aliases", () => { + const fs = MemoryFileSystem.create( + new SharedArrayBuffer(4 * 1024 * 1024), + ); + addDinitBaseSystemFiles(fs); + + const source = readFileSync( + join(findRepoRoot(), "images", "rootfs", "etc", "services"), + "utf8", + ); + const derived = readGuestFile(fs, "/etc/services"); + + expect(derived).toBe(source); + expect(derived).toContain("www www-http"); + expect(derived).toContain("postgresql\t5432/tcp"); + }); +}); diff --git a/host/test/dylink.test.ts b/host/test/dylink.test.ts index 32ca00438..b493423e3 100644 --- a/host/test/dylink.test.ts +++ b/host/test/dylink.test.ts @@ -4,6 +4,7 @@ import { describe, it, expect } from "vitest"; import { + createLongjmpTag, parseDylinkSection, loadSharedLibrary, loadSharedLibrarySync, @@ -70,13 +71,14 @@ function buildDylinkWat( forkCapabilities?: number, tableSize = 0, memorySize = 0, + wat2wasmFlags: string[] = [], ): Uint8Array { const dir = join(tmpdir(), "wasm-dylink-wat-test"); mkdirSync(dir, { recursive: true }); const watPath = join(dir, `${name}.wat`); const wasmPath = join(dir, `${name}.wasm`); writeFileSync(watPath, wat); - execFileSync("wat2wasm", ["--enable-threads", watPath, "-o", wasmPath], { + execFileSync("wat2wasm", ["--enable-threads", ...wat2wasmFlags, watPath, "-o", wasmPath], { stdio: "pipe", }); const module = new Uint8Array(readFileSync(wasmPath)); @@ -103,6 +105,103 @@ function buildDylinkWat( ); } +describe.skipIf(typeof WebAssembly.Tag !== "function")("longjmp tag identity", () => { + const cases = [ + { ptrWidth: 4 as const, wasmType: "i32", value: 37 }, + { ptrWidth: 8 as const, wasmType: "i64", value: 37n }, + ]; + + it.each(cases)( + "shares one process-owned $wasmType tag with a side module", + ({ ptrWidth, wasmType, value }) => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $longjmp (import "env" "__c_longjmp") (param ${wasmType})) + (func (export "throw_longjmp") (param $value ${wasmType}) + local.get $value + throw $longjmp)) + `, `longjmp-${wasmType}`, undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + const longjmpTag = createLongjmpTag(ptrWidth)!; + options.ptrWidth = ptrWidth; + options.longjmpTag = longjmpTag; + // A same-named main export is not authoritative for this reserved tag. + options.globalSymbols.set("__c_longjmp", () => 0); + + const lib = loadSharedLibrarySync(`liblongjmp-${wasmType}.so`, wasmBytes, options); + const throwLongjmp = lib.exports.throw_longjmp as (arg: number | bigint) => void; + let caught: unknown; + try { + throwLongjmp(value); + } catch (error) { + caught = error; + } + + const WasmException = (WebAssembly as typeof WebAssembly & { + Exception: new (...args: unknown[]) => Error; + }).Exception; + expect(caught).toBeInstanceOf(WasmException); + const exception = caught as Error & { + is: (tag: WebAssembly.Tag) => boolean; + getArg: (tag: WebAssembly.Tag, index: number) => unknown; + }; + expect(exception.is(longjmpTag)).toBe(true); + expect(exception.getArg(longjmpTag, 0)).toBe(value); + }, + ); + + it.each(cases)( + "creates one pointer-width-aware $wasmType fallback for standalone linkers", + ({ ptrWidth, wasmType, value }) => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $longjmp (import "env" "__c_longjmp") (param ${wasmType})) + (func (export "throw_longjmp") (param $value ${wasmType}) + local.get $value + throw $longjmp)) + `, `fallback-longjmp-${wasmType}`, undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + options.ptrWidth = ptrWidth; + + const first = loadSharedLibrarySync(`libfallback-${wasmType}-one.so`, wasmBytes, options); + const fallbackTag = options.longjmpTag!; + expect(fallbackTag).toBeInstanceOf(WebAssembly.Tag); + const second = loadSharedLibrarySync(`libfallback-${wasmType}-two.so`, wasmBytes, options); + expect(options.longjmpTag).toBe(fallbackTag); + + for (const lib of [first, second]) { + let caught: unknown; + try { + (lib.exports.throw_longjmp as (arg: number | bigint) => void)(value); + } catch (error) { + caught = error; + } + const exception = caught as { + is: (tag: WebAssembly.Tag) => boolean; + getArg: (tag: WebAssembly.Tag, index: number) => unknown; + }; + expect(exception.is(fallbackTag)).toBe(true); + expect(exception.getArg(fallbackTag, 0)).toBe(value); + } + }, + ); + + it("rejects a lookalike tag before side-module instantiation", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $longjmp (import "env" "__c_longjmp") (param i32))) + `, "invalid-longjmp-tag", undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + options.longjmpTag = {} as WebAssembly.Tag; + + expect(() => loadSharedLibrarySync("libinvalid-longjmp.so", wasmBytes, options)) + .toThrow(/__c_longjmp must be an actual WebAssembly\.Tag/); + }); +}); + describe.skipIf(!hasCompiler())("dylink.0 parser", () => { it("parses a simple shared library", () => { const wasmBytes = buildSharedLib( diff --git a/host/test/exec-state-tracking.test.ts b/host/test/exec-state-tracking.test.ts index 55c08205b..a8ed46812 100644 --- a/host/test/exec-state-tracking.test.ts +++ b/host/test/exec-state-tracking.test.ts @@ -9,6 +9,7 @@ import { CH_ARGS, CH_DATA_SIZE, CH_RETURN, + HOST_INTERCEPTED_SYSCALLS, } from "../src/generated/abi"; describe("exec host-state transition", () => { @@ -249,8 +250,8 @@ describe("exec host-state transition", () => { bytes.set(path, pathPtr); const blobPtr = 0x200; bytes.fill(0, blobPtr, blobPtr + 40); - let resolveProgram!: (value: ArrayBuffer) => void; - const program = new Promise((resolve) => { + let resolveProgram!: (value: ReturnType) => void; + const program = new Promise>((resolve) => { resolveProgram = resolve; }); const kernelSpawn = vi.fn(() => 100); @@ -270,7 +271,7 @@ describe("exec host-state transition", () => { worker.handleSpawn(channel, [pathPtr, path.length, blobPtr, 40, 0, 0]); worker.processes.get(7).channels = []; - resolveProgram(new ArrayBuffer(8)); + resolveProgram(resolvedProgram()); await Promise.resolve(); await Promise.resolve(); @@ -279,6 +280,46 @@ describe("exec host-state transition", () => { expect(completeChannel).not.toHaveBeenCalled(); }); + it("rejects an unlaunchable spawn before creating a child or applying file actions", async () => { + const memory = new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }); + const channel = createChannel(7, memory, 0); + const bytes = new Uint8Array(memory.buffer); + const pathPtr = 0x100; + const path = new TextEncoder().encode("/bin/malformed"); + bytes.set(path, pathPtr); + const blobPtr = 0x200; + bytes.fill(0, blobPtr, blobPtr + 40); + const kernelSpawn = vi.fn(() => 100); + const onSpawn = vi.fn(async () => 0); + const completeChannel = vi.fn(); + const worker = createWorker({ + processes: new Map([[7, { channels: [channel], memory }]]), + callbacks: { + onResolveSpawn: vi.fn(async () => ({ errno: 8 })), + onSpawn, + }, + completeChannel, + kernelInstance: { + exports: { kernel_spawn_process: kernelSpawn }, + }, + }); + + worker.handleSpawn(channel, [pathPtr, path.length, blobPtr, 40, 0, 0]); + await Promise.resolve(); + await Promise.resolve(); + + expect(kernelSpawn).not.toHaveBeenCalled(); + expect(onSpawn).not.toHaveBeenCalled(); + expect(completeChannel).toHaveBeenCalledWith( + channel, + HOST_INTERCEPTED_SYSCALLS.SYS_SPAWN, + [pathPtr, path.length, blobPtr, 40, 0, 0], + undefined, + -1, + 8, + ); + }); + it("keeps a created spawn child but suppresses stale parent completion", async () => { const memory = new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }); const channel = createChannel(7, memory, 0); @@ -289,9 +330,11 @@ describe("exec host-state transition", () => { const kernelSpawn = vi.fn(() => 100); const removeProcess = vi.fn(); const completeChannel = vi.fn(); + const onSpawn = vi.fn(() => spawned); + const program = resolvedProgram(); const worker = createWorker({ processes: new Map([[7, { channels: [channel], memory }]]), - callbacks: { onSpawn: vi.fn(() => spawned) }, + callbacks: { onSpawn }, completeChannel, kernelMemory: new WebAssembly.Memory({ initial: 1 }), scratchOffset: 0, @@ -311,10 +354,10 @@ describe("exec host-state transition", () => { 0, new Uint8Array(40), 40, + program, [], - [], - new ArrayBuffer(8), ); + expect(onSpawn).toHaveBeenCalledWith(100, program, []); worker.processes.get(7).channels = []; finishSpawn(0); await Promise.resolve(); @@ -370,9 +413,8 @@ describe("exec host-state transition", () => { 0, new Uint8Array(40), 40, + resolvedProgram(), [], - [], - new ArrayBuffer(8), ); expect(worker.tcpListenerTargets.get(8080)).toContainEqual({ @@ -1036,6 +1078,18 @@ function createWorker(overrides: Record): any { }); } +function resolvedProgram() { + const programBytes = Uint8Array.from([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + ]).buffer; + return { + programBytes, + programModule: new WebAssembly.Module(programBytes), + argv: [], + }; +} + function createChannel(pid: number, memory: WebAssembly.Memory, channelOffset: number): any { return { pid, diff --git a/host/test/exec.test.ts b/host/test/exec.test.ts index 5726624ad..34743b979 100644 --- a/host/test/exec.test.ts +++ b/host/test/exec.test.ts @@ -2,6 +2,9 @@ * Tests for execve support — loading a new program binary into an existing process. */ import { describe, it, expect } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { runCentralizedProgram } from "./centralized-test-helper"; import { tryResolveBinary } from "../src/binary-resolver"; @@ -59,4 +62,35 @@ describe("execve", () => { expect(result.stdout).toContain("argv[1]=from-fork"); expect(result.stdout).toContain("FROM=fork"); }); + + it.skipIf(!hasExecCaller)("rejects malformed Wasm before committing exec", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "kandelo-exec-malformed-wasm-")); + const malformedWasm = join(tempDir, "malformed.wasm"); + try { + // Valid Wasm magic/version with a truncated type section. This reaches + // compilation, which must fail before kernelExecSetup discards the old + // process image. + writeFileSync(malformedWasm, Buffer.from([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + 0x01, 0x01, 0xff, + ])); + + const result = await runCentralizedProgram({ + programPath: execCallerBinary!, + argv: ["exec-caller"], + timeout: 15_000, + execPrograms: new Map([ + ["/bin/exec-child", malformedWasm], + ]), + }); + + expect(result.exitCode).toBe(127); + expect(result.stderr).toContain("Exec format error"); + expect(result.stderr).not.toContain("Centralized worker failed"); + expect(result.stderr).not.toContain("WebAssembly.compile()"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); }); diff --git a/host/test/fetch-backend.test.ts b/host/test/fetch-backend.test.ts index b13274a72..21610509a 100644 --- a/host/test/fetch-backend.test.ts +++ b/host/test/fetch-backend.test.ts @@ -4,6 +4,7 @@ import { TlsNetworkBackend, type TlsMitmConnection } from "../src/networking/tls const encoder = new TextEncoder(); const decoder = new TextDecoder(); +const MSG_PEEK = 0x0002; afterEach(() => { vi.restoreAllMocks(); @@ -83,13 +84,25 @@ class LoopbackMitmTls implements TlsMitmConnection { async close(): Promise {} } +async function waitForReadable( + backend: Pick, + handle: number, +): Promise { + const deadline = Date.now() + 1_000; + while (Date.now() < deadline) { + if ((backend.poll(handle, 0x0001) & 0x0001) !== 0) return; + await new Promise((resolve) => setTimeout(resolve, 0)); + } + throw new Error("timed out waiting for readable poll"); +} + describe("FetchNetworkBackend", () => { afterEach(() => { vi.restoreAllMocks(); }); describe("getaddrinfo", () => { - it("returns a 4-byte address for any hostname", () => { + it("returns a 4-byte address for DNS names that can be deferred to fetch", () => { const backend = new FetchNetworkBackend(); const addr = backend.getaddrinfo("example.com"); expect(addr.length).toBe(4); @@ -125,6 +138,19 @@ describe("FetchNetworkBackend", () => { expect(() => backend.getaddrinfo(".toto.toto.toto")).toThrow("ENOENT"); expect(() => backend.getaddrinfo(`www.${"x".repeat(100)}.com`)).toThrow("ENOENT"); }); + + it("rejects the reserved invalid zone without rejecting unqualified names", () => { + const backend = new FetchNetworkBackend(); + expect(backend.getaddrinfo("dummy-host-name").length).toBe(4); + expect(() => backend.getaddrinfo("totes.invalid")).toThrow("ENOENT"); + }); + + it("allows explicitly aliased unqualified names", () => { + const backend = new FetchNetworkBackend({ + hostAliases: { registry: "registry.npmjs.org" }, + }); + expect(backend.getaddrinfo("registry").length).toBe(4); + }); }); describe("connect", () => { @@ -168,6 +194,32 @@ describe("FetchNetworkBackend", () => { }); }); + it("honors MSG_PEEK without consuming buffered response bytes", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("hello"))); + const backend = new FetchNetworkBackend(); + const addr = backend.getaddrinfo("example.com"); + backend.connect(1, addr, 80); + backend.send( + 1, + encoder.encode("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"), + 0, + ); + + const first = decoder.decode(await recvWhenReady(backend, 1)); + expect(first).toContain("hello"); + + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("world"))); + backend.send( + 1, + encoder.encode("GET /2 HTTP/1.1\r\nHost: example.com\r\n\r\n"), + 0, + ); + await waitForReadable(backend, 1); + const peeked = decoder.decode(backend.recv(1, 4, MSG_PEEK)); + const consumed = decoder.decode(backend.recv(1, 4, 0)); + expect(peeked).toBe(consumed); + }); + describe("hostAliases", () => { it("rewrites the fetch target while preserving the request port", () => { const fetchMock = vi @@ -216,6 +268,19 @@ describe("TlsNetworkBackend HTTP proxy path", () => { expect(() => backend.getaddrinfo(".toto.toto.toto")).toThrow("ENOENT"); expect(() => backend.getaddrinfo(`www.${"x".repeat(100)}.com`)).toThrow("ENOENT"); }); + + it("rejects special-use invalid but permits potentially resolvable unqualified names", () => { + const backend = new TlsNetworkBackend(); + expect(backend.getaddrinfo("dummy-host-name").length).toBe(4); + expect(() => backend.getaddrinfo("totes.invalid")).toThrow("ENOENT"); + }); + + it("allows explicitly aliased unqualified names", () => { + const backend = new TlsNetworkBackend({ + dnsAliases: { registry: "https://registry.npmjs.org" }, + }); + expect(backend.getaddrinfo("registry").length).toBe(4); + }); }); it("resets response state for keep-alive HTTP requests", async () => { @@ -267,10 +332,56 @@ describe("TlsNetworkBackend HTTP proxy path", () => { expect(response.toLowerCase()).not.toContain("content-encoding"); expect(response.toLowerCase()).not.toContain("connection: close"); }); + + it("routes HTTP fetches through the configured CORS proxy", async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response("proxied")); + vi.stubGlobal("fetch", fetchMock); + const proxyPrefix = "https://kandelo.test/proxy?url="; + const backend = new TlsNetworkBackend({ + corsProxyUrl: proxyPrefix, + dnsAliases: {}, + }); + const addr = backend.getaddrinfo("example.com"); + backend.connect(1, addr, 80); + + backend.send( + 1, + encoder.encode("GET /resource HTTP/1.1\r\nHost: example.com\r\n\r\n"), + 0, + ); + await recvWhenReady(backend, 1); + + expect(fetchMock).toHaveBeenCalledWith( + `${proxyPrefix}${encodeURIComponent("http://example.com/resource")}`, + expect.any(Object), + ); + }); + + it("honors MSG_PEEK without consuming HTTP response bytes", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("peek-body"))); + const backend = new TlsNetworkBackend(); + const addr = backend.getaddrinfo("proxy.local"); + backend.connect(1, addr, 80); + + sendGet(backend, 1, "/peek"); + await recvWhenReady(backend, 1); + + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("second-body"))); + sendGet(backend, 1, "/peek2"); + await recvWhenReady(backend, 1); + + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("third-body"))); + sendGet(backend, 1, "/peek3"); + const peeked = decoder.decode((await recvWhenReady({ + recv: (handle, maxLen) => backend.recv(handle, maxLen, MSG_PEEK), + }, 1)).subarray(0, 8)); + const consumed = decoder.decode(backend.recv(1, 8, 0)); + expect(peeked).toBe(consumed); + }); }); describe("TlsNetworkBackend TLS MITM path", () => { - it("delivers the full response to recv() before reporting EOF", async () => { + it("polls and peeks encrypted response bytes before reporting EOF", async () => { const body = "mitm-response-body"; vi.stubGlobal( "fetch", @@ -293,8 +404,39 @@ describe("TlsNetworkBackend TLS MITM path", () => { .getWriter() .write(encoder.encode("GET /readme HTTP/1.1\r\nHost: example.com\r\n\r\n")); - const response = decoder.decode(await recvWhenReady(backend, 1)); + await waitForReadable(backend, 1); + const peeked = backend.recv(1, 8, MSG_PEEK); + expect(backend.poll(1, 0x0001) & 0x0001).toBe(0x0001); + const consumed = backend.recv(1, 8, 0); + expect(peeked).toEqual(consumed); + const response = decoder.decode( + new Uint8Array([...consumed, ...await recvWhenReady(backend, 1)]), + ); expect(response).toContain("200"); expect(response).toContain(body); }); + + it("routes decrypted HTTPS requests through the configured CORS proxy", async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response("proxied TLS")); + vi.stubGlobal("fetch", fetchMock); + const proxyPrefix = "https://kandelo.test/proxy?"; + let tls!: LoopbackMitmTls; + const backend = new TlsNetworkBackend({ + corsProxyUrl: proxyPrefix, + createTlsConnection: () => (tls = new LoopbackMitmTls()), + }); + await backend.init(); + + const addr = backend.getaddrinfo("example.com"); + backend.connect(2, addr, 443); + await tls.serverEnd.upstream.writable + .getWriter() + .write(encoder.encode("GET /secure HTTP/1.1\r\nHost: example.com\r\n\r\n")); + await waitForReadable(backend, 2); + + expect(fetchMock).toHaveBeenCalledWith( + `${proxyPrefix}https://example.com/secure`, + expect.any(Object), + ); + }); }); diff --git a/host/test/fixtures/deep-wasm-recursion-worker.mjs b/host/test/fixtures/deep-wasm-recursion-worker.mjs new file mode 100644 index 000000000..72ce50e58 --- /dev/null +++ b/host/test/fixtures/deep-wasm-recursion-worker.mjs @@ -0,0 +1,7 @@ +import { readFileSync } from "node:fs"; +import { parentPort, workerData } from "node:worker_threads"; + +const bytes = readFileSync(workerData.wasmPath); +const { instance } = await WebAssembly.instantiate(bytes); +const recurse = instance.exports.recurse; +parentPort.postMessage({ result: recurse(workerData.depth) }); diff --git a/host/test/fixtures/deep-wasm-recursion.wat b/host/test/fixtures/deep-wasm-recursion.wat new file mode 100644 index 000000000..4be3073ea --- /dev/null +++ b/host/test/fixtures/deep-wasm-recursion.wat @@ -0,0 +1,14 @@ +(module + (func $recurse (export "recurse") (param $depth i32) (result i32) + local.get $depth + i32.eqz + if (result i32) + i32.const 0 + else + local.get $depth + i32.const 1 + i32.sub + call $recurse + i32.const 1 + i32.add + end)) diff --git a/host/test/fixtures/sharedfs-lock-release-worker.mjs b/host/test/fixtures/sharedfs-lock-release-worker.mjs new file mode 100644 index 000000000..f190f235f --- /dev/null +++ b/host/test/fixtures/sharedfs-lock-release-worker.mjs @@ -0,0 +1,10 @@ +import { parentPort, workerData } from "node:worker_threads"; + +const words = new Int32Array(workerData); +const namespaceLockIndex = 64 / Int32Array.BYTES_PER_ELEMENT; +Atomics.store(words, namespaceLockIndex, 1); +parentPort.postMessage("locked"); +setTimeout(() => { + Atomics.store(words, namespaceLockIndex, 0); + Atomics.notify(words, namespaceLockIndex); +}, 100); diff --git a/host/test/fixtures/sharedfs-namespace-worker.ts b/host/test/fixtures/sharedfs-namespace-worker.ts new file mode 100644 index 000000000..0eb7fd623 --- /dev/null +++ b/host/test/fixtures/sharedfs-namespace-worker.ts @@ -0,0 +1,57 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { MemoryFileSystem } from "../../src/vfs/memory-fs"; + +const { fsBuffer, controlBuffer, role, iterations } = workerData as { + fsBuffer: SharedArrayBuffer; + controlBuffer: SharedArrayBuffer; + role: "mutator" | "observer"; + iterations: number; +}; +const control = new Int32Array(controlBuffer); +const fs = MemoryFileSystem.fromExisting(fsBuffer); +const O_RDONLY = 0; +const O_WRONLY = 1; +const O_CREAT = 0x40; +const O_TRUNC = 0x200; + +while (Atomics.load(control, 0) === 0) Atomics.wait(control, 0, 0); + +try { + if (role === "mutator") { + for (let i = 0; i < iterations; i++) { + try { fs.unlink("/slot"); } catch { /* observer may hold an unlinked fd */ } + + let fd = fs.open("/other", O_WRONLY | O_CREAT | O_TRUNC, 0o644); + fs.write(fd, new Uint8Array([0x22]), null, 1); + fs.close(fd); + fs.unlink("/other"); + + fd = fs.open("/slot", O_WRONLY | O_CREAT | O_TRUNC, 0o644); + fs.write(fd, new Uint8Array([0x11]), null, 1); + fs.close(fd); + } + } else { + for (let i = 0; i < iterations; i++) { + let fd: number | null = null; + try { + fd = fs.open("/slot", O_RDONLY, 0); + const byte = new Uint8Array(1); + const read = fs.read(fd, byte, null, 1); + if (read === 1 && byte[0] !== 0x11) { + throw new Error(`path ABA exposed data from recycled inode: ${byte[0]}`); + } + } catch (error) { + if (error instanceof Error && error.message.includes("path ABA")) throw error; + // ENOENT is valid between unlink and recreate. + } finally { + if (fd !== null) fs.close(fd); + } + } + } + parentPort!.postMessage({ ok: true }); +} catch (error) { + parentPort!.postMessage({ + ok: false, + error: error instanceof Error ? error.stack ?? error.message : String(error), + }); +} diff --git a/host/test/getpwent.test.ts b/host/test/getpwent.test.ts index e7508b06e..1eb38a7eb 100644 --- a/host/test/getpwent.test.ts +++ b/host/test/getpwent.test.ts @@ -91,7 +91,7 @@ describe.skipIf(!haveSmoke || !haveRootfs)("getpwent via rootfs.vfs mount", () = expect(result.stdout).toContain("GRENT count=8"); }); - it("resolves service names from /etc/services via getservbyname", async () => { + it("resolves canonical service names and aliases from the rootfs image", async () => { const result = await runCentralizedProgram({ programPath: smokeWasm, argv: ["getpwent_smoke"], @@ -101,5 +101,12 @@ describe.skipIf(!haveSmoke || !haveRootfs)("getpwent via rootfs.vfs mount", () = expect(result.exitCode, result.stderr || result.stdout).toBe(0); expect(result.stdout).toContain("SERV name=ssh proto=tcp port=22"); expect(result.stdout).toContain("SERV name=http proto=tcp port=80"); + expect(result.stdout).toContain("SERVENT query=www name=www proto=tcp port=80"); + expect(result.stdout).toContain("SERVENT query=www-http name=www-http proto=tcp port=80"); + expect(result.stdout).toContain("SERVENT query=https name=https proto=tcp port=443"); + expect(result.stdout).toContain("SERVENT query=mysql name=mysql proto=tcp port=3306"); + expect(result.stdout).toContain( + "SERVENT query=postgresql name=postgresql proto=tcp port=5432", + ); }); }); diff --git a/host/test/global-setup.ts b/host/test/global-setup.ts index b46f1d1c6..f512821e5 100644 --- a/host/test/global-setup.ts +++ b/host/test/global-setup.ts @@ -21,6 +21,7 @@ const fixturesDir = join(__dirname, "fixtures"); /** C programs that tests depend on. */ const TEST_PROGRAMS = [ + "clock_getcpuclockid_test.c", "putenv_test.c", "getaddrinfo_test.c", "sysv_ipc_test.c", @@ -42,8 +43,12 @@ const TEST_PROGRAMS = [ "thread-exit-group.c", ]; -/** WAT fixtures used by host/test/wasi-shim.test.ts. */ -const WAT_FIXTURES = ["wasi-args.wat", "wasi-hello.wat"]; +/** WAT fixtures used by host runtime tests. */ +const WAT_FIXTURES = [ + "deep-wasm-recursion.wat", + "wasi-args.wat", + "wasi-hello.wat", +]; function needsRebuild(srcFile: string, outFile: string): boolean { if (!existsSync(outFile)) return true; diff --git a/host/test/host-diagnostic-routing.test.ts b/host/test/host-diagnostic-routing.test.ts new file mode 100644 index 000000000..ae7662aef --- /dev/null +++ b/host/test/host-diagnostic-routing.test.ts @@ -0,0 +1,34 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "../.."); + +const entries = [ + ["Node", join(repoRoot, "host/src/node-kernel-worker-entry.ts")], + ["browser", join(repoRoot, "host/src/browser-kernel-worker-entry.ts")], +] as const; + +describe.each(entries)("%s kernel-worker diagnostic routing", (_name, path) => { + const source = readFileSync(path, "utf8"); + + it("reserves stderr protocol messages for the kernel's real onStderr bytes", () => { + const stderrPosts = source.match(/type:\s*"stderr"/g) ?? []; + expect(stderrPosts).toHaveLength(1); + expect(source).toMatch(/onStderr:[\s\S]{0,160}type:\s*"stderr"/); + }); + + it("routes lifecycle, protocol, exec, clone, and thread failures as host diagnostics", () => { + for (const diagnosticSource of [ + "worker protocol", + "worker-main error message", + "exec post-commit transition", + "clone allocation", + "thread worker failure", + ]) { + expect(source).toContain(`source: "${diagnosticSource}"`); + } + expect(source).toContain("reportHostDiagnostic({"); + }); +}); diff --git a/host/test/ifhwaddr.test.ts b/host/test/ifhwaddr.test.ts index 08c4eae00..e71e0c3cb 100644 --- a/host/test/ifhwaddr.test.ts +++ b/host/test/ifhwaddr.test.ts @@ -1,31 +1,61 @@ import { describe, it, expect } from "vitest"; import { runCentralizedProgram } from "./centralized-test-helper"; import { resolveBinary } from "../src/binary-resolver"; +import { LocalVirtualNetwork } from "../src/networking/virtual-network"; +import { NodePlatformIO } from "../src/platform/node"; -describe("SIOCGIFCONF / SIOCGIFHWADDR", () => { - it("returns a virtual MAC address via ioctl", async () => { - const result = await runCentralizedProgram({ - programPath: resolveBinary("programs/ifhwaddr.wasm"), - timeout: 10_000, - }); +describe("virtual network interface ioctls", () => { + it.each([ + { + arch: "wasm32", + programPath: "programs/ifhwaddr.wasm", + ifreqSize: 32, + }, + { + arch: "wasm64", + programPath: "programs/wasm64/ifhwaddr.wasm", + ifreqSize: 40, + }, + ])( + "honors interface and guest-memory contracts for $arch", + async ({ arch, programPath, ifreqSize }) => { + const network = new LocalVirtualNetwork(); + const io = new NodePlatformIO(); + io.network = network.attachMachine({ + id: `ifhwaddr-${arch}`, + address: [10, 23, 45, 67], + }); - expect(result.exitCode).toBe(0); + const result = await runCentralizedProgram({ + programPath: resolveBinary(programPath), + io, + timeout: 30_000, + }); - // Should find one interface named "eth0" - expect(result.stdout).toContain("interfaces: 1"); - expect(result.stdout).toContain("name: eth0"); + expect(result.exitCode).toBe(0); + expect(result.stdout).not.toContain("FAIL:"); + expect(result.stdout).toContain(`ifreq-size: ${ifreqSize}`); + expect(result.stdout).toContain("ifconf: lo=127.0.0.1 eth0=10.23.45.67"); + expect(result.stdout).toContain("eth0-address: 10.23.45.67"); + expect(result.stdout).toContain( + "libc-name-to-index: lo=1 eth0=2 missing=0 errno=19", + ); + expect(result.stdout).toContain("libc-invalid-index: errno=6"); + expect(result.stdout).toContain("nameindex: lo=1"); + expect(result.stdout).toContain("nameindex: eth0=2"); + expect(result.stdout).toContain( + "PASS: virtual interface ioctl and libc contracts", + ); - // MAC should be locally-administered and non-zero - expect(result.stdout).toContain("locally-administered: yes"); - expect(result.stdout).toContain("non-zero: yes"); + const macMatch = result.stdout.match( + /eth0-mac: ([0-9a-f]{2}(?::[0-9a-f]{2}){5})/, + ); + expect(macMatch).not.toBeNull(); - // MAC format: xx:xx:xx:xx:xx:xx - const macMatch = result.stdout.match(/mac: ([0-9a-f]{2}(?::[0-9a-f]{2}){5})/); - expect(macMatch).not.toBeNull(); - - // Verify locally-administered bit (second-lowest bit of first octet) - const firstOctet = parseInt(macMatch![1].split(":")[0], 16); - expect(firstOctet & 0x02).toBe(0x02); // locally administered - expect(firstOctet & 0x01).toBe(0x00); // unicast - }); + const firstOctet = parseInt(macMatch![1].split(":")[0], 16); + expect(firstOctet & 0x02).toBe(0x02); + expect(firstOctet & 0x01).toBe(0x00); + }, + 30_000, + ); }); diff --git a/host/test/lazy-archive.test.ts b/host/test/lazy-archive.test.ts index 5e18ba65a..218860ac8 100644 --- a/host/test/lazy-archive.test.ts +++ b/host/test/lazy-archive.test.ts @@ -5,6 +5,8 @@ import { parseZipCentralDirectory } from "../src/vfs/zip"; import type { ZipEntry } from "../src/vfs/zip"; const O_RDONLY = 0x0000; +const O_WRONLY = 0x0001; +const O_TRUNC = 0x0200; function createMemfs(): MemoryFileSystem { const sab = new SharedArrayBuffer(4 * 1024 * 1024); @@ -61,9 +63,170 @@ function makeRealZip() { return { zipBytes, entries: parseZipCentralDirectory(zipBytes) }; } +function makeTwoMemberZip() { + const zipBytes = zipSync({ + "a.txt": new TextEncoder().encode("alpha"), + "b.txt": new TextEncoder().encode("bravo"), + }); + return { zipBytes, entries: parseZipCentralDirectory(zipBytes) }; +} + +function readText(mfs: MemoryFileSystem, path: string): string { + const fd = mfs.open(path, O_RDONLY, 0); + const buffer = new Uint8Array(64); + const read = mfs.read(fd, buffer, null, buffer.length); + mfs.close(fd); + return new TextDecoder().decode(buffer.subarray(0, read)); +} + // --- Task 3: Registration --- describe("Lazy archive group registration", () => { + it("atomically replaces an existing file with archive backing", async () => { + const originalFetch = globalThis.fetch; + const mfs = createMemfs(); + const { zipBytes, entries } = makeRealZip(); + mfs.mkdir("/opt", 0o755); + mfs.mkdir("/opt/bin", 0o755); + mfs.createFileWithOwner("/opt/bin/hello", 0o644, 0, 0, new Uint8Array([9])); + + mfs.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + try { + await expect(mfs.ensureMaterialized("/opt/bin/hello")).resolves.toBe( + true, + ); + expect(readText(mfs, "/opt/bin/hello")).toBe("#!/bin/sh\necho hello"); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("does not overwrite a peer write after replacing an archive path", async () => { + const originalFetch = globalThis.fetch; + const mfs = createMemfs(); + const peer = MemoryFileSystem.fromExisting(mfs.sharedBuffer); + const { zipBytes, entries } = makeRealZip(); + mfs.mkdir("/opt", 0o755); + mfs.mkdir("/opt/bin", 0o755); + mfs.createFileWithOwner("/opt/bin/hello", 0o644, 0, 0, new Uint8Array([4])); + + const raw = ( + mfs as unknown as { + fs: { + createLazyStub: ( + path: string, + mode: number, + ) => { ino: number; generation: number; dataSequence: number }; + }; + } + ).fs; + const createLazyStub = raw.createLazyStub.bind(raw); + const createSpy = vi + .spyOn(raw, "createLazyStub") + .mockImplementation((path, mode) => { + const identity = createLazyStub(path, mode); + if (path === "/opt/bin/hello") { + const writer = peer.open(path, O_WRONLY, 0o644); + peer.write(writer, new Uint8Array([9]), null, 1); + peer.close(writer); + } + return identity; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + try { + mfs.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + await expect(mfs.ensureMaterialized("/opt/bin/hello")).resolves.toBe( + true, + ); + const fd = mfs.open("/opt/bin/hello", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(mfs.read(fd, byte, null, 1)).toBe(1); + mfs.close(fd); + expect(byte[0]).toBe(9); + } finally { + createSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("replaces stale standalone backing when a path moves into an archive", async () => { + const originalFetch = globalThis.fetch; + const mfs = createMemfs(); + const zipBytes = zipSync({ item: new Uint8Array([7]) }); + mfs.registerLazyFile("/item", "http://example.com/old", 1); + mfs.registerLazyArchiveFromEntries( + "http://example.com/archive.zip", + parseZipCentralDirectory(zipBytes), + "/", + ); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + globalThis.fetch = fetchMock; + + try { + await expect(mfs.ensureMaterialized("/item")).resolves.toBe(true); + expect(fetchMock).toHaveBeenCalledWith("http://example.com/archive.zip"); + const fd = mfs.open("/item", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(mfs.read(fd, byte, null, 1)).toBe(1); + mfs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("replaces stale archive backing when a member becomes standalone", async () => { + const originalFetch = globalThis.fetch; + const mfs = createMemfs(); + const zipBytes = zipSync({ item: new Uint8Array([7]) }); + mfs.registerLazyArchiveFromEntries( + "http://example.com/archive.zip", + parseZipCentralDirectory(zipBytes), + "/", + ); + mfs.registerLazyFile("/item", "http://example.com/standalone", 1); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([8]).buffer), + } as unknown as Response); + globalThis.fetch = fetchMock; + + try { + expect(mfs.exportLazyArchiveEntries()).toEqual([]); + await expect(mfs.ensureMaterialized("/item")).resolves.toBe(true); + expect(fetchMock).toHaveBeenCalledWith("http://example.com/standalone"); + const fd = mfs.open("/item", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(mfs.read(fd, byte, null, 1)).toBe(1); + mfs.close(fd); + expect(byte[0]).toBe(8); + } finally { + globalThis.fetch = originalFetch; + } + }); + it("registerLazyArchiveFromEntries creates stubs for all files", () => { const mfs = createMemfs(); const entries = makeFakeEntries(); @@ -404,6 +567,85 @@ describe("Lazy archive materialization", () => { const st = mfs.stat("/opt/bin/hello"); expect(st.mode & 0o777).toBe(0o700); }); + + it("keeps a peer-renamed member lazy when another archive member materializes", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const peer = MemoryFileSystem.fromExisting(sab); + const { zipBytes, entries } = makeTwoMemberZip(); + + owner.registerLazyArchiveFromEntries( + "http://example.com/two-members.zip", + entries, + "/pkg", + ); + peer.rename("/pkg/b.txt", "/pkg/moved-b.txt"); + + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + await expect(owner.ensureMaterialized("/pkg/a.txt")).resolves.toBe(true); + expect(readText(owner, "/pkg/a.txt")).toBe("alpha"); + + await expect(owner.ensureMaterialized("/pkg/moved-b.txt")).resolves.toBe( + true, + ); + expect(readText(owner, "/pkg/moved-b.txt")).toBe("bravo"); + }); + + it("finishes a requested archive member renamed during its fetch", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const peer = MemoryFileSystem.fromExisting(sab); + const { zipBytes, entries } = makeTwoMemberZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/two-members.zip", + entries, + "/pkg", + ); + let release!: (value: ArrayBuffer) => void; + const body = new Promise((resolve) => { + release = resolve; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => body, + } as unknown as Response); + + const pending = owner.ensureMaterialized("/pkg/a.txt"); + await vi.waitFor(() => expect(globalThis.fetch).toHaveBeenCalled()); + peer.rename("/pkg/a.txt", "/pkg/moved-a.txt"); + release(zipBytes.buffer); + + await expect(pending).resolves.toBe(true); + expect(readText(owner, "/pkg/moved-a.txt")).toBe("alpha"); + }); + + it("materializes pending archives into a self-contained image", async () => { + const mfs = createMemfs(); + const { zipBytes, entries } = makeRealZip(); + mfs.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + globalThis.fetch = fetchMock; + + const restored = MemoryFileSystem.fromImage( + await mfs.saveImage({ materializeAll: true }), + ); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(restored.exportLazyArchiveEntries()).toEqual([]); + expect(readText(restored, "/opt/bin/hello")).toBe("#!/bin/sh\necho hello"); + expect(readText(restored, "/opt/share/data.txt")).toBe("hello world"); + }); }); // --- Task 5: Unlink tracking --- @@ -467,6 +709,60 @@ describe("Lazy archive unlink tracking", () => { expect(new TextDecoder().decode(buf.subarray(0, n))).toBe("hello world"); }); + it("retains a peer-created hard-link alias after unlink", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const peer = MemoryFileSystem.fromExisting(sab); + const { zipBytes, entries } = makeRealZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + peer.link("/opt/bin/hello", "/archive-alias"); + + owner.unlink("/opt/bin/hello"); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + await expect(owner.ensureMaterialized("/archive-alias")).resolves.toBe( + true, + ); + expect(readText(owner, "/archive-alias")).toBe("#!/bin/sh\necho hello"); + }); + + it("retains a peer-created archive alias when rename replaces its other name", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const peer = MemoryFileSystem.fromExisting(sab); + const { zipBytes, entries } = makeRealZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + peer.link("/opt/bin/hello", "/archive-alias"); + owner.createFileWithOwner("/source", 0o644, 0, 0, new Uint8Array([4])); + + owner.rename("/source", "/opt/bin/hello"); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + await expect(owner.ensureMaterialized("/archive-alias")).resolves.toBe( + true, + ); + expect(readText(owner, "/archive-alias")).toBe("#!/bin/sh\necho hello"); + const destination = owner.open("/opt/bin/hello", O_RDONLY, 0); + const destinationByte = new Uint8Array(1); + expect(owner.read(destination, destinationByte, null, 1)).toBe(1); + owner.close(destination); + expect(destinationByte[0]).toBe(4); + }); + it("unlink of non-archive file does not affect archive groups", () => { const mfs = createMemfs(); const entries = makeFakeEntries(); @@ -530,6 +826,65 @@ describe("Lazy archive export/import", () => { expect(mfs2.stat("/usr/share/vim/README").size).toBe(0); }); + it("retains the original archive member name when importing legacy metadata", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const { zipBytes, entries } = makeRealZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + + const serialized = owner.exportLazyArchiveEntries(); + for (const entry of serialized[0].entries) delete entry.archivePath; + + const restoredMetadata = MemoryFileSystem.fromExisting(sab); + restoredMetadata.importLazyArchiveEntries(serialized); + restoredMetadata.rename("/opt/bin/hello", "/opt/bin/moved"); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + await expect( + restoredMetadata.ensureMaterialized("/opt/bin/moved"), + ).resolves.toBe(true); + expect(readText(restoredMetadata, "/opt/bin/moved")).toBe( + "#!/bin/sh\necho hello", + ); + }); + + it("rejects sequence-less archive metadata after a live peer write", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const { entries } = makeRealZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + const serialized = owner.exportLazyArchiveEntries(); + const hello = serialized[0].entries.find( + (entry) => entry.vfsPath === "/opt/bin/hello", + )!; + delete hello.dataSequence; + + const writer = owner.open("/opt/bin/hello", O_WRONLY | O_TRUNC, 0o644); + owner.write(writer, new Uint8Array([9]), null, 1); + owner.close(writer); + + const peer = MemoryFileSystem.fromExisting(sab); + expect(() => peer.importLazyArchiveEntries(serialized)).toThrow( + /requires inode generation and data sequence/, + ); + + expect(peer.stat("/opt/bin/hello").size).toBe(1); + await expect(peer.ensureMaterialized("/opt/bin/hello")).resolves.toBe( + false, + ); + }); + it("rebaseToNewFileSystem preserves unmaterialized archive metadata", () => { const mfs = createMemfs(); const entries = makeFakeEntries(); @@ -557,7 +912,7 @@ describe("Lazy archive export/import", () => { rebased.close(fd); }); - it("import of materialized group does not populate lazyArchiveInodes", async () => { + it("omits fully materialized groups from deferred archive metadata", async () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs1 = MemoryFileSystem.create(sab); const { zipBytes, entries } = makeRealZip(); @@ -576,10 +931,9 @@ describe("Lazy archive export/import", () => { await mfs1.ensureMaterialized("/opt/bin/hello"); - // Export from instance 1 — group should be materialized + // Concrete files no longer need archive metadata in another instance. const serialized = mfs1.exportLazyArchiveEntries(); - expect(serialized).toHaveLength(1); - expect(serialized[0].materialized).toBe(true); + expect(serialized).toEqual([]); // Import into instance 2 on the same SAB const mfs2 = MemoryFileSystem.fromExisting(sab); diff --git a/host/test/node-host-vfs-only-metadata.test.ts b/host/test/node-host-vfs-only-metadata.test.ts index c5b0ee8b3..f0d3bc77e 100644 --- a/host/test/node-host-vfs-only-metadata.test.ts +++ b/host/test/node-host-vfs-only-metadata.test.ts @@ -272,6 +272,26 @@ describe.each(backendFactories)("%s", (_name, makeCase) => { }); }); +describe("HostFileSystem default virtual ownership", () => { + it("can present existing host-backed files as owned by a chosen guest uid/gid", () => { + const root = makeTempRoot("wasm-posix-host-fs-default-owner-"); + const native = join(root, "owned-by-mount"); + writeFileSync(native, "data"); + const before = statSync(native); + + const backend = new HostFileSystem(root, "/", { uid: 65534, gid: 65533 }); + const virtual = backend.stat("/owned-by-mount"); + expect(virtual.uid).toBe(65534); + expect(virtual.gid).toBe(65533); + + backend.chown("/owned-by-mount", 1000, 1001); + const changed = backend.stat("/owned-by-mount"); + expect(changed.uid).toBe(1000); + expect(changed.gid).toBe(1001); + expectNativeMetadataUnchanged(native, before); + }); +}); + describe("VirtualPlatformIO on Node host mounts", () => { it("routes metadata operations to HostFileSystem as VFS-only changes", () => { const root = makeTempRoot("wasm-posix-virtual-platform-vfs-only-"); diff --git a/host/test/node-kernel-host-diagnostic.test.ts b/host/test/node-kernel-host-diagnostic.test.ts new file mode 100644 index 000000000..0440e45ee --- /dev/null +++ b/host/test/node-kernel-host-diagnostic.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from "vitest"; +import { NodeKernelHost } from "../src/node-kernel-host"; +import type { KernelToMainMessage } from "../src/node-kernel-protocol"; + +interface TestableNodeKernelHost { + handleWorkerMessage(message: KernelToMainMessage): void; +} + +describe("NodeKernelHost diagnostics", () => { + it("delivers host diagnostics without invoking the guest stderr callback", () => { + const onHostDiagnostic = vi.fn(); + const onStderr = vi.fn(); + const host = new NodeKernelHost({ onHostDiagnostic, onStderr }); + const testable = host as unknown as TestableNodeKernelHost; + + testable.handleWorkerMessage({ + type: "host_diagnostic", + pid: 42, + source: "clone allocation", + message: "host allocation failed", + }); + + expect(onHostDiagnostic).toHaveBeenCalledOnce(); + expect(onHostDiagnostic).toHaveBeenCalledWith({ + pid: 42, + source: "clone allocation", + message: "host allocation failed", + }); + expect(onStderr).not.toHaveBeenCalled(); + }); + + it("keeps actual stderr messages on the guest callback", () => { + const onHostDiagnostic = vi.fn(); + const onStderr = vi.fn(); + const host = new NodeKernelHost({ onHostDiagnostic, onStderr }); + const testable = host as unknown as TestableNodeKernelHost; + const data = new TextEncoder().encode("guest stderr\n"); + + testable.handleWorkerMessage({ type: "stderr", pid: 7, data }); + + expect(onStderr).toHaveBeenCalledWith(7, data); + expect(onHostDiagnostic).not.toHaveBeenCalled(); + }); +}); diff --git a/host/test/pthread-trap-semantics.test.ts b/host/test/pthread-trap-semantics.test.ts index 7cd6b76d3..6105928e1 100644 --- a/host/test/pthread-trap-semantics.test.ts +++ b/host/test/pthread-trap-semantics.test.ts @@ -30,7 +30,7 @@ describe.skipIf(!hasPthreadTrapBinaries)("pthread trap POSIX semantics", () => { }, 15_000); it("terminates the process when a pthread worker hits an uncaught guest trap", async () => { - const { exitCode, stderr } = await runCentralizedProgram({ + const { exitCode, stderr, hostDiagnostics } = await runCentralizedProgram({ programPath: trapChildBinary, argv: ["pthread-trap-child"], timeout: 10_000, @@ -38,7 +38,9 @@ describe.skipIf(!hasPthreadTrapBinaries)("pthread trap POSIX semantics", () => { }); expect(stderr).toContain("pthread-trap-child: before trap"); - expect(stderr).toContain("Thread worker failed"); + expect(stderr).not.toContain("Thread worker failed"); + expect(hostDiagnostics.map((entry) => entry.message).join("\n")) + .toContain("Thread worker failed"); expect(stderr).not.toContain("FAIL pthread_join returned"); expect(exitCode).toBe(signalExitStatus(SIGILL)); }, 15_000); diff --git a/host/test/sharedfs-safety.test.ts b/host/test/sharedfs-safety.test.ts new file mode 100644 index 000000000..557e70441 --- /dev/null +++ b/host/test/sharedfs-safety.test.ts @@ -0,0 +1,937 @@ +import { describe, expect, it, vi } from "vitest"; +import { Worker } from "node:worker_threads"; +import { MemoryFileSystem } from "../src/vfs/memory-fs"; + +const O_RDONLY = 0x0000; +const O_WRONLY = 0x0001; +const O_RDWR = 0x0002; +const O_CREAT = 0x0040; +const O_TRUNC = 0x0200; +const SEEK_SET = 0; + +function create(size = 4 * 1024 * 1024): MemoryFileSystem { + return MemoryFileSystem.create(new SharedArrayBuffer(size)); +} + +function listDir(fs: MemoryFileSystem, path: string): string[] { + const dd = fs.opendir(path); + const names: string[] = []; + try { + for (;;) { + const entry = fs.readdir(dd); + if (!entry) return names; + if (entry.name !== "." && entry.name !== "..") names.push(entry.name); + } + } finally { + fs.closedir(dd); + } +} + +describe("SharedFS sparse-file safety", () => { + it("extends a multi-gigabyte sparse file without scanning its holes", () => { + const fs = create(); + const fd = fs.open("/sparse", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const sparseSize = 3_000_000_000; + + const started = performance.now(); + fs.ftruncate(fd, sparseSize); + expect(performance.now() - started).toBeLessThan(500); + expect(fs.fstat(fd).size).toBe(sparseSize); + + fs.seek(fd, sparseSize - 1, SEEK_SET); + expect(fs.write(fd, new Uint8Array([0x7a]), null, 1)).toBe(1); + const tail = new Uint8Array(4); + expect(fs.read(fd, tail, sparseSize - 3, tail.length)).toBe(3); + expect(Array.from(tail.subarray(0, 3))).toEqual([0, 0, 0x7a]); + fs.close(fd); + }); + + it("rejects invalid and unrepresentable truncate lengths", () => { + const fs = create(); + const fd = fs.open("/file", O_CREAT | O_RDWR, 0o644); + expect(() => fs.ftruncate(fd, -1)).toThrow(/Invalid argument/); + expect(() => fs.ftruncate(fd, Number.NaN)).toThrow(/Invalid argument/); + expect(() => fs.ftruncate(fd, Number.MAX_SAFE_INTEGER)).toThrow( + /File too large/, + ); + expect(fs.fstat(fd).size).toBe(0); + fs.close(fd); + }); + + it("commits size and data for a positive partial write at ENOSPC", () => { + const fs = create(128 * 1024); + const fd = fs.open("/partial", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const input = new Uint8Array(1024 * 1024).fill(0xa5); + const written = fs.write(fd, input, null, input.length); + + expect(written).toBeGreaterThan(0); + expect(written).toBeLessThan(input.length); + expect(fs.fstat(fd).size).toBe(written); + const tail = new Uint8Array(1); + expect(fs.read(fd, tail, written - 1, 1)).toBe(1); + expect(tail[0]).toBe(0xa5); + fs.close(fd); + }); +}); + +describe("SharedFS namespace and image safety", () => { + it("hard-links a symlink inode without following its target", () => { + const fs = create(); + const target = fs.open("/target", O_CREAT | O_WRONLY, 0o644); + fs.close(target); + fs.symlink("/target", "/symbolic"); + + fs.link("/symbolic", "/alias"); + expect(fs.lstat("/alias").mode & 0xf000).toBe(0xa000); + expect(fs.lstat("/alias").nlink).toBe(2); + expect(fs.readlink("/alias")).toBe("/target"); + + fs.unlink("/symbolic"); + expect(fs.lstat("/alias").mode & 0xf000).toBe(0xa000); + expect(fs.lstat("/alias").nlink).toBe(1); + }); + + it("prevents path ABA across workers without deadlocking", async () => { + const fs = create(); + const initial = fs.open("/slot", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(initial, new Uint8Array([0x11]), null, 1); + fs.close(initial); + + const controlBuffer = new SharedArrayBuffer(4); + const control = new Int32Array(controlBuffer); + const workerUrl = new URL( + "./fixtures/sharedfs-namespace-worker.ts", + import.meta.url, + ); + const makeWorker = (role: "mutator" | "observer") => + new Worker(workerUrl, { + execArgv: ["--import", "tsx"], + workerData: { + fsBuffer: fs.sharedBuffer, + controlBuffer, + role, + iterations: 4_000, + }, + }); + const workers = [makeWorker("mutator"), makeWorker("observer")]; + const results = workers.map( + (worker) => + new Promise<{ ok: boolean; error?: string }>((resolve, reject) => { + worker.once("message", resolve); + worker.once("error", reject); + worker.once("exit", (code) => { + if (code !== 0) reject(new Error(`SharedFS worker exited ${code}`)); + }); + }), + ); + + Atomics.store(control, 0, 1); + Atomics.notify(control, 0, workers.length); + try { + const completed = Promise.all(results); + const timeout = new Promise((_, reject) => { + setTimeout( + () => reject(new Error("SharedFS worker watchdog expired")), + 8_000, + ); + }); + const messages = await Promise.race([completed, timeout]); + expect(messages).toEqual([{ ok: true }, { ok: true }]); + } finally { + await Promise.all(workers.map((worker) => worker.terminate())); + } + }, 10_000); + + it("polls safely when browser-main-style Atomics.wait is unavailable", async () => { + const fs = create(); + const worker = new Worker( + new URL("./fixtures/sharedfs-lock-release-worker.mjs", import.meta.url), + { workerData: fs.sharedBuffer }, + ); + await new Promise((resolve, reject) => { + worker.once("message", () => resolve()); + worker.once("error", reject); + }); + + const wait = vi.spyOn(Atomics, "wait").mockImplementation(() => { + throw new TypeError("Atomics.wait cannot be called on this thread"); + }); + try { + expect(fs.stat("/").mode & 0xf000).toBe(0x4000); + expect(wait).toHaveBeenCalled(); + } finally { + wait.mockRestore(); + await worker.terminate(); + } + }); + + it("leaves a full directory intact after repeated create and rename failures", () => { + const fs = create(128 * 1024); + const filler = fs.open("/filler", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(filler, new Uint8Array(1024 * 1024), null, 1024 * 1024); + fs.close(filler); + + const created: string[] = []; + let failedName = ""; + for (let i = 0; i < 64; i++) { + const name = `/${String(i).padStart(2, "0")}-${"x".repeat(180)}`; + try { + const fd = fs.open(name, O_CREAT | O_WRONLY, 0o644); + fs.close(fd); + created.push(name); + } catch (error) { + expect(String(error)).toMatch(/No space left/); + failedName = name; + break; + } + } + + expect(failedName).not.toBe(""); + const before = listDir(fs, "/").sort(); + for (let attempt = 0; attempt < 3; attempt++) { + expect(() => fs.open(failedName, O_CREAT | O_WRONLY, 0o644)).toThrow( + /No space left/, + ); + expect(listDir(fs, "/").sort()).toEqual(before); + } + + const source = created[0]; + const destination = `/${"r".repeat(220)}`; + expect(() => fs.rename(source, destination)).toThrow(/No space left/); + expect(fs.stat(source).mode & 0xf000).toBe(0x8000); + expect(() => fs.stat(destination)).toThrow(/No such file/); + expect(listDir(fs, "/").sort()).toEqual(before); + }); + + it("requires quiescent snapshots and clears legacy runtime state on restore", async () => { + const fs = create(); + const fd = fs.open("/saved", O_CREAT | O_RDWR, 0o644); + await expect(fs.saveImage()).rejects.toThrow(/open descriptors/); + fs.close(fd); + const orphanFd = fs.open("/orphan", O_CREAT | O_RDWR, 0o644); + const orphanIno = fs.fstat(orphanFd).ino; + fs.close(orphanFd); + + const image = await fs.saveImage(); + const imageView = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); + const sabOffset = 16; + imageView.setUint32(sabOffset + 60, 1, true); // stale grow lock + imageView.setUint32(sabOffset + 64, 1, true); // stale namespace lock + imageView.setUint32(sabOffset + 256, 1, true); // stale fd 0 + + const inodeTableBlock = imageView.getUint32(sabOffset + 36, true); + const ino = fs.stat("/saved").ino; + const inodeOffset = sabOffset + inodeTableBlock * 4096 + (ino % 32) * 128; + imageView.setUint32(inodeOffset, 0x80000000, true); // stale inode lock + imageView.setUint32(inodeOffset + 112, 1, true); // stale open ref + + const orphanOffset = + sabOffset + inodeTableBlock * 4096 + (orphanIno % 32) * 128; + imageView.setUint32(orphanOffset + 12, 0, true); // unlinked + imageView.setUint32(orphanOffset + 112, 1, true); // but held open + imageView.setUint32(sabOffset + 256 + 4, orphanIno, true); + + const rootOffset = sabOffset + inodeTableBlock * 4096 + 128; + const rootBlock = imageView.getUint32(rootOffset + 48, true); + const rootSize = Number(imageView.getBigUint64(rootOffset + 16, true)); + const decoder = new TextDecoder(); + for (let pos = 0; pos < rootSize;) { + const entryOffset = sabOffset + rootBlock * 4096 + pos; + const recLen = imageView.getUint16(entryOffset + 4, true); + const nameLen = imageView.getUint16(entryOffset + 6, true); + const name = decoder.decode( + image.subarray(entryOffset + 8, entryOffset + 8 + nameLen), + ); + if (name === "orphan") imageView.setUint32(entryOffset, 0, true); + pos += recLen; + } + + const restored = MemoryFileSystem.fromImage(image); + expect(restored.stat("/saved").ino).toBe(ino); + expect(() => restored.stat("/orphan")).toThrow(/No such file/); + const replacement = restored.open( + "/replacement", + O_CREAT | O_WRONLY, + 0o644, + ); + expect(restored.fstat(replacement).ino).toBe(orphanIno); + restored.close(replacement); + expect(() => restored.close(0)).toThrow(/Bad file descriptor/); + await expect(restored.saveImage()).resolves.toBeInstanceOf(Uint8Array); + }); + + it("fails closed instead of removing a corrupt directory", () => { + const fs = create(); + fs.mkdir("/empty", 0o755); + const ino = fs.stat("/empty").ino; + const view = new DataView(fs.sharedBuffer); + const inodeTableBlock = view.getUint32(36, true); + const inodeOffset = inodeTableBlock * 4096 + (ino % 32) * 128; + const dataBlock = view.getUint32(inodeOffset + 48, true); + const recLenOffset = dataBlock * 4096 + 4; + const originalRecLen = view.getUint16(recLenOffset, true); + + view.setUint16(recLenOffset, 0, true); + expect(() => fs.rmdir("/empty")).toThrow(/I\/O error/); + expect(fs.stat("/empty").ino).toBe(ino); + + view.setUint16(recLenOffset, originalRecLen, true); + fs.rmdir("/empty"); + expect(() => fs.stat("/empty")).toThrow(/No such file/); + }); + + it("rejects a directory entry that names a free inode slot", () => { + const fs = create(); + const fd = fs.open("/victim", O_CREAT | O_WRONLY, 0o644); + fs.close(fd); + const view = new DataView(fs.sharedBuffer); + const inodeTableBlock = view.getUint32(36, true); + const rootOffset = inodeTableBlock * 4096 + 128; + const rootBlock = view.getUint32(rootOffset + 48, true); + const rootSize = Number(view.getBigUint64(rootOffset + 16, true)); + const decoder = new TextDecoder(); + let victimEntry = -1; + for (let pos = 0; pos < rootSize;) { + const abs = rootBlock * 4096 + pos; + const recLen = view.getUint16(abs + 4, true); + const nameLen = view.getUint16(abs + 6, true); + const name = decoder.decode( + new Uint8Array(fs.sharedBuffer, abs + 8, nameLen), + ); + if (name === "victim") victimEntry = abs; + pos += recLen; + } + expect(victimEntry).toBeGreaterThan(0); + + const freeInode = view.getUint32(16, true) - 1; + view.setUint32(victimEntry, freeInode, true); + const freeBefore = fs.statfs("/").ffree; + expect(() => fs.stat("/victim")).toThrow(/I\/O error/); + expect(() => fs.unlink("/victim")).toThrow(/I\/O error/); + expect(fs.statfs("/").ffree).toBe(freeBefore); + }); +}); + +describe("MemoryFileSystem lazy inode identity", () => { + it("atomically replaces an existing file with lazy backing", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const existing = fs.open("/lazy", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(existing, new Uint8Array([9]), null, 1); + fs.close(existing); + + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + expect(fs.getLazyEntry("/lazy")).toMatchObject({ size: 1 }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/lazy")).resolves.toBe(true); + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("does not overwrite a peer write after replacing an existing lazy path", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + const existing = fs.open("/lazy", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(existing, new Uint8Array([4]), null, 1); + fs.close(existing); + + const raw = ( + fs as unknown as { + fs: { + createLazyStub: ( + path: string, + mode: number, + ) => { ino: number; generation: number; dataSequence: number }; + }; + } + ).fs; + const createLazyStub = raw.createLazyStub.bind(raw); + const createSpy = vi + .spyOn(raw, "createLazyStub") + .mockImplementation((path, mode) => { + const identity = createLazyStub(path, mode); + const writer = peer.open(path, O_WRONLY, 0o644); + peer.write(writer, new Uint8Array([9]), null, 1); + peer.close(writer); + return identity; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + expect(fs.getLazyEntry("/lazy")).toBeNull(); + await expect(fs.ensureMaterialized("/lazy")).resolves.toBe(false); + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(9); + } finally { + createSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("binds registration to the atomically-created stub identity", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + type StubIdentity = ReturnType<(typeof fs)["registerLazyFile"]>; + const raw = ( + fs as unknown as { + fs: { + createLazyStub: ( + path: string, + mode: number, + ) => { + ino: StubIdentity; + generation: number; + dataSequence: number; + }; + }; + } + ).fs; + const createLazyStub = raw.createLazyStub.bind(raw); + const createSpy = vi + .spyOn(raw, "createLazyStub") + .mockImplementation((path, mode) => { + const identity = createLazyStub(path, mode); + peer.rename(path, "/moved"); + const replacement = peer.open(path, O_CREAT | O_WRONLY, 0o644); + peer.write(replacement, new Uint8Array([9]), null, 1); + peer.close(replacement); + return identity; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + expect(fs.getLazyEntry("/lazy")).toBeNull(); + expect(fs.getLazyEntry("/moved")).not.toBeNull(); + await expect(fs.ensureMaterialized("/moved")).resolves.toBe(true); + const replacement = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(replacement, byte, null, 1)).toBe(1); + fs.close(replacement); + expect(byte[0]).toBe(9); + } finally { + createSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("rejects generation-less lazy metadata from a live peer", () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 999); + const [legacy] = fs.exportLazyEntries(); + delete legacy.generation; + delete legacy.dataSequence; + fs.unlink("/lazy"); + const replacement = fs.open("/lazy", O_CREAT | O_WRONLY, 0o644); + fs.close(replacement); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + expect(() => peer.importLazyEntries([legacy])).toThrow( + /requires inode generation and data sequence/, + ); + + expect(peer.getLazyEntry("/lazy")).toBeNull(); + expect(peer.stat("/lazy").size).toBe(0); + }); + + it("rejects sequence-less lazy metadata after same-inode content changes", () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 999); + const [legacy] = fs.exportLazyEntries(); + delete legacy.dataSequence; + + const writer = fs.open("/lazy", O_WRONLY | O_TRUNC, 0o644); + fs.write(writer, new Uint8Array([9]), null, 1); + fs.close(writer); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + expect(() => peer.importLazyEntries([legacy])).toThrow( + /requires inode generation and data sequence/, + ); + + expect(peer.getLazyEntry("/lazy")).toBeNull(); + expect(peer.stat("/lazy").size).toBe(1); + }); + + it("does not apply lazy metadata after an inode slot is recycled", () => { + const fs = create(); + const oldIno = fs.registerLazyFile( + "/lazy", + "https://example.test/lazy", + 999, + ); + const exported = fs.exportLazyEntries(); + fs.unlink("/lazy"); + + const fd = fs.open("/replacement", O_CREAT | O_WRONLY, 0o644); + fs.close(fd); + expect(fs.stat("/replacement").ino).toBe(oldIno); + expect(fs.stat("/replacement").size).toBe(0); + expect(fs.getLazyEntry("/replacement")).toBeNull(); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + peer.importLazyEntries(exported); + expect(peer.stat("/replacement").size).toBe(0); + expect(peer.getLazyEntry("/replacement")).toBeNull(); + }); + + it("tracks lazy files across rename and hard-link lifecycle", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 3); + fs.rename("/lazy", "/renamed"); + fs.link("/renamed", "/alias"); + fs.unlink("/renamed"); + + expect(fs.getLazyEntry("/alias")).toMatchObject({ + path: "/alias", + size: 3, + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "3" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([1, 2, 3]).buffer), + } as unknown as Response); + try { + await expect(fs.ensureMaterialized("/alias")).resolves.toBe(true); + } finally { + globalThis.fetch = originalFetch; + } + expect(fs.stat("/alias").size).toBe(3); + }); + + it("retains a peer-created hard-link alias after unlink", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + peer.link("/lazy", "/alias"); + + fs.unlink("/lazy"); + expect(fs.getLazyEntry("/alias")).toMatchObject({ + path: "/alias", + paths: ["/alias"], + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/alias")).resolves.toBe(true); + const fd = fs.open("/alias", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("retains a peer-created lazy alias when rename replaces its other name", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + fs.registerLazyFile("/destination", "https://example.test/lazy", 1); + peer.link("/destination", "/alias"); + const source = fs.open("/source", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(source, new Uint8Array([4]), null, 1); + fs.close(source); + + fs.rename("/source", "/destination"); + expect(fs.getLazyEntry("/alias")).toMatchObject({ path: "/alias" }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/alias")).resolves.toBe(true); + const alias = fs.open("/alias", O_RDONLY, 0); + const aliasByte = new Uint8Array(1); + expect(fs.read(alias, aliasByte, null, 1)).toBe(1); + fs.close(alias); + expect(aliasByte[0]).toBe(7); + + const destination = fs.open("/destination", O_RDONLY, 0); + const destinationByte = new Uint8Array(1); + expect(fs.read(destination, destinationByte, null, 1)).toBe(1); + fs.close(destination); + expect(destinationByte[0]).toBe(4); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("uses the identity actually removed when unlink races a peer rename", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + const raw = ( + fs as unknown as { + fs: { unlink: (path: string) => unknown }; + } + ).fs; + const unlink = raw.unlink.bind(raw); + const unlinkSpy = vi.spyOn(raw, "unlink").mockImplementation((path) => { + peer.rename(path, "/moved"); + const replacement = peer.open(path, O_CREAT | O_WRONLY, 0o644); + peer.write(replacement, new Uint8Array([9]), null, 1); + peer.close(replacement); + return unlink(path); + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + fs.unlink("/lazy"); + expect(fs.getLazyEntry("/moved")).not.toBeNull(); + await expect(fs.ensureMaterialized("/moved")).resolves.toBe(true); + } finally { + unlinkSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("uses the source actually renamed when a peer replaces the old path", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + const raw = ( + fs as unknown as { + fs: { rename: (oldPath: string, newPath: string) => unknown }; + } + ).fs; + const rename = raw.rename.bind(raw); + const renameSpy = vi + .spyOn(raw, "rename") + .mockImplementation((oldPath, newPath) => { + peer.rename(oldPath, "/moved"); + const replacement = peer.open(oldPath, O_CREAT | O_WRONLY, 0o644); + peer.write(replacement, new Uint8Array([9]), null, 1); + peer.close(replacement); + return rename(oldPath, newPath); + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + fs.rename("/lazy", "/renamed"); + expect(fs.getLazyEntry("/moved")).not.toBeNull(); + expect(fs.stat("/renamed").size).toBe(1); + await expect(fs.ensureMaterialized("/moved")).resolves.toBe(true); + } finally { + renameSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("preserves every lazy hard-link name across instance transfer", () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 3); + fs.link("/lazy", "/alias"); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + peer.importLazyEntries(fs.exportLazyEntries()); + peer.unlink("/lazy"); + + expect(peer.getLazyEntry("/alias")).toMatchObject({ + path: "/alias", + paths: ["/alias"], + size: 3, + }); + }); + + it("preserves ordinary and lazy hard-link identity while rebasing", () => { + const fs = create(); + const regular = fs.open("/regular", O_CREAT | O_WRONLY, 0o644); + fs.write(regular, new Uint8Array([1, 2, 3]), null, 3); + fs.close(regular); + fs.link("/regular", "/regular-alias"); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 3); + fs.link("/lazy", "/lazy-alias"); + + const rebased = fs.rebaseToNewFileSystem(8 * 1024 * 1024); + expect(rebased.stat("/regular").ino).toBe( + rebased.stat("/regular-alias").ino, + ); + expect(rebased.stat("/regular").nlink).toBe(2); + expect(rebased.stat("/lazy").ino).toBe(rebased.stat("/lazy-alias").ino); + expect(rebased.stat("/lazy").nlink).toBe(2); + expect(rebased.getLazyEntry("/lazy-alias")).toMatchObject({ + size: 3, + paths: expect.arrayContaining(["/lazy", "/lazy-alias"]), + }); + }); + + it("rebases from one coherent snapshot when a peer renames afterward", () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 3); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + + // Inject the peer mutation immediately after SharedFS captures its bytes + // and identity map. The rebased result must consistently reflect that + // captured pre-rename state rather than walking the newer live tree. + const shared = ( + fs as unknown as { + fs: { snapshotState: () => unknown }; + } + ).fs; + const snapshotState = shared.snapshotState.bind(shared); + shared.snapshotState = () => { + const snapshot = snapshotState(); + peer.rename("/lazy", "/moved"); + return snapshot; + }; + + const rebased = fs.rebaseToNewFileSystem(8 * 1024 * 1024); + expect(peer.stat("/moved").size).toBe(0); + expect(() => peer.stat("/lazy")).toThrow(); + expect(rebased.stat("/lazy").size).toBe(3); + expect(() => rebased.stat("/moved")).toThrow(); + expect(rebased.getLazyEntry("/lazy")).toMatchObject({ + path: "/lazy", + size: 3, + }); + }); + + it("drops deferred backing after an explicit write through a hard link", async () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 999); + fs.link("/lazy", "/alias"); + + const fd = fs.open("/alias", O_WRONLY | O_TRUNC, 0o644); + expect(fs.write(fd, new Uint8Array([0x5a]), null, 1)).toBe(1); + fs.close(fd); + + expect(fs.stat("/lazy").size).toBe(1); + expect(fs.getLazyEntry("/lazy")).toBeNull(); + expect(fs.getLazyEntry("/alias")).toBeNull(); + expect(fs.exportLazyEntries()).toEqual([]); + await expect(fs.ensureMaterialized("/alias")).resolves.toBe(false); + }); + + it("does not apply a delayed fetch to a replacement inode", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + let release!: (value: ArrayBuffer) => void; + const body = new Promise((resolve) => { + release = resolve; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => body, + } as unknown as Response); + + try { + const pending = fs.ensureMaterialized("/lazy"); + await vi.waitFor(() => expect(globalThis.fetch).toHaveBeenCalled()); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + peer.unlink("/lazy"); + const replacement = peer.open("/lazy", O_CREAT | O_WRONLY, 0o644); + peer.write(replacement, new Uint8Array([9]), null, 1); + peer.close(replacement); + + release(new Uint8Array([1]).buffer); + await expect(pending).resolves.toBe(false); + + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(9); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("finishes one materialization call after a peer rename during fetch", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + let release!: (value: ArrayBuffer) => void; + const body = new Promise((resolve) => { + release = resolve; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => body, + } as unknown as Response); + + try { + const pending = fs.ensureMaterialized("/lazy"); + await vi.waitFor(() => expect(globalThis.fetch).toHaveBeenCalled()); + peer.rename("/lazy", "/moved"); + release(new Uint8Array([7]).buffer); + + await expect(pending).resolves.toBe(true); + const fd = fs.open("/moved", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + expect(fs.getLazyEntry("/moved")).toBeNull(); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("keeps an empty lazy stub retryable after ENOSPC", async () => { + const originalFetch = globalThis.fetch; + const fs = create(256 * 1024); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 4096); + const filler = fs.open("/filler", O_CREAT | O_WRONLY, 0o644); + const chunk = new Uint8Array(64 * 1024).fill(0xa5); + while (fs.write(filler, chunk, null, chunk.length) > 0) { + // Fill every allocatable block. + } + fs.close(filler); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "4096" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array(4096).fill(7).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/lazy")).rejects.toThrow( + /No space left/, + ); + expect(fs.getLazyEntry("/lazy")).not.toBeNull(); + expect(fs.stat("/lazy").size).toBe(4096); + + fs.unlink("/filler"); + await expect(fs.ensureMaterialized("/lazy")).resolves.toBe(true); + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("does not overwrite a same-inode write that wins after fetch", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([1]).buffer), + } as unknown as Response); + + type ReplaceArgs = [string, number, number, number, Uint8Array]; + const raw = ( + fs as unknown as { + fs: { replaceIfIdentity: (...args: ReplaceArgs) => boolean }; + } + ).fs; + const replace = raw.replaceIfIdentity.bind(raw); + const replaceSpy = vi + .spyOn(raw, "replaceIfIdentity") + .mockImplementation((...args: ReplaceArgs) => { + const fd = peer.open("/lazy", O_WRONLY | O_TRUNC, 0o644); + peer.write(fd, new Uint8Array([9]), null, 1); + peer.close(fd); + return replace(...args); + }); + + try { + await expect(fs.ensureMaterialized("/lazy")).resolves.toBe(false); + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(9); + } finally { + replaceSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("materializes through a surviving path renamed by another instance", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + fs.symlink("/moved", "/indirect"); + peer.rename("/lazy", "/moved"); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/indirect")).resolves.toBe(true); + const fd = fs.open("/moved", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); diff --git a/host/test/symlink.test.ts b/host/test/symlink.test.ts index f848dd677..a5674d40a 100644 --- a/host/test/symlink.test.ts +++ b/host/test/symlink.test.ts @@ -51,6 +51,22 @@ describe("symlink and lstat", () => { expect(st.mode & S_IFMT).toBe(S_IFLNK); }); + it("unlink removes a dangling symlink itself", () => { + const mfs = createMemfs(); + + const fd = mfs.open("/target.txt", O_WRONLY | O_CREAT | O_TRUNC, 0o644); + mfs.close(fd); + mfs.symlink("target.txt", "/link.txt"); + + mfs.unlink("/target.txt"); + expect(mfs.lstat("/link.txt").mode & S_IFMT).toBe(S_IFLNK); + + // POSIX unlink(2) unlinks the directory entry named by path. When path is + // a symlink, it removes the link inode and does not follow the target. + mfs.unlink("/link.txt"); + expect(() => mfs.lstat("/link.txt")).toThrow(); + }); + it("readlink returns the symlink target", () => { const mfs = createMemfs(); mfs.symlink("/some/path", "/mylink"); diff --git a/host/test/tcp-backend.test.ts b/host/test/tcp-backend.test.ts index cda08a9be..83f2fddb8 100644 --- a/host/test/tcp-backend.test.ts +++ b/host/test/tcp-backend.test.ts @@ -3,6 +3,8 @@ import * as net from "node:net"; import { TcpNetworkBackend } from "../src/networking/tcp-backend"; const LOOPBACK = new Uint8Array([127, 0, 0, 1]); +const POLLIN = 0x0001; +const MSG_PEEK = 0x0002; async function listenLoopback(): Promise<{ server: net.Server; @@ -48,6 +50,15 @@ async function waitForConnected(backend: TcpNetworkBackend, handle: number): Pro throw new Error("connect timed out"); } +async function waitForReadable(backend: TcpNetworkBackend, handle: number): Promise { + const deadline = Date.now() + 2_000; + while (Date.now() < deadline) { + if ((backend.poll(handle, POLLIN) & POLLIN) !== 0) return; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error("readable data timed out"); +} + describe("TcpNetworkBackend hostname parsing", () => { it.each([ ["2130706433", [127, 0, 0, 1]], @@ -94,6 +105,35 @@ describe("TcpNetworkBackend", () => { await expect(accepted).resolves.toEqual({ data: "hello", ended: true }); }); + + it("honors MSG_PEEK on real Node TCP sockets", async () => { + let acceptedSocket!: net.Socket; + let resolveAccepted!: () => void; + const accepted = new Promise((resolve) => { resolveAccepted = resolve; }); + const server = net.createServer((socket) => { + acceptedSocket = socket; + resolveAccepted(); + }); + servers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + + const backend = new TcpNetworkBackend(); + backend.connect(10, LOOPBACK, (server.address() as net.AddressInfo).port); + await waitForConnected(backend, 10); + await accepted; + acceptedSocket.write("peek-data"); + await waitForReadable(backend, 10); + + expect(new TextDecoder().decode(backend.recv(10, 4, MSG_PEEK))).toBe("peek"); + expect(new TextDecoder().decode(backend.recv(10, 9, 0))).toBe("peek-data"); + + backend.close(10); + acceptedSocket.destroy(); + }); + it("keeps the real writable half open after peer FIN", async () => { let acceptedSocket!: net.Socket; let resolveAccepted!: () => void; diff --git a/host/test/time-provider.test.ts b/host/test/time-provider.test.ts new file mode 100644 index 000000000..cf26053cf --- /dev/null +++ b/host/test/time-provider.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { NodePlatformIO } from "../src/platform/node"; +import { BrowserTimeProvider, NodeTimeProvider } from "../src/vfs/time"; + +function asNanoseconds(value: { sec: number; nsec: number }): bigint { + return BigInt(value.sec) * 1_000_000_000n + BigInt(value.nsec); +} + +describe.each([ + ["NodeTimeProvider", () => new NodeTimeProvider()], + ["BrowserTimeProvider", () => new BrowserTimeProvider()], + ["NodePlatformIO", () => new NodePlatformIO()], +] as const)("%s CLOCK_BOOTTIME", (_name, createProvider) => { + it("uses the same nondecreasing domain as CLOCK_MONOTONIC", () => { + const provider = createProvider(); + const monotonic = asNanoseconds(provider.clockGettime(1)); + const boottime = asNanoseconds(provider.clockGettime(7)); + + expect(boottime).toBeGreaterThanOrEqual(monotonic); + expect(boottime - monotonic).toBeLessThan(100_000_000n); + }); +}); diff --git a/host/test/vfs-image.test.ts b/host/test/vfs-image.test.ts index 44cca1457..c028d8672 100644 --- a/host/test/vfs-image.test.ts +++ b/host/test/vfs-image.test.ts @@ -18,13 +18,22 @@ function createMemfs(): MemoryFileSystem { return MemoryFileSystem.create(sab); } -function writeFile(mfs: MemoryFileSystem, path: string, data: Uint8Array, mode = 0o644): void { +function writeFile( + mfs: MemoryFileSystem, + path: string, + data: Uint8Array, + mode = 0o644, +): void { // Ensure parent directories exist const parts = path.split("/").filter(Boolean); let current = ""; for (let i = 0; i < parts.length - 1; i++) { current += "/" + parts[i]; - try { mfs.mkdir(current, 0o755); } catch { /* exists */ } + try { + mfs.mkdir(current, 0o755); + } catch { + /* exists */ + } } const fd = mfs.open(path, O_WRONLY | O_CREAT | O_TRUNC, mode); mfs.write(fd, data, null, data.length); @@ -52,6 +61,28 @@ function readDir(mfs: MemoryFileSystem, path: string): string[] { return names.sort(); } +function stripStandaloneLazyIdentity(image: Uint8Array): Uint8Array { + const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const sabLen = view.getUint32(12, true); + const lazyOffset = 16 + sabLen; + const lazyLen = view.getUint32(lazyOffset, true); + const entries = JSON.parse( + new TextDecoder().decode( + image.subarray(lazyOffset + 4, lazyOffset + 4 + lazyLen), + ), + ) as Array<{ generation?: number; dataSequence?: number }>; + for (const entry of entries) { + delete entry.generation; + delete entry.dataSequence; + } + const lazyJson = new TextEncoder().encode(JSON.stringify(entries)); + const legacy = new Uint8Array(lazyOffset + 4 + lazyJson.byteLength); + legacy.set(image.subarray(0, lazyOffset)); + new DataView(legacy.buffer).setUint32(lazyOffset, lazyJson.byteLength, true); + legacy.set(lazyJson, lazyOffset + 4); + return legacy; +} + describe("VFS image save/restore", () => { describe("saveImage + fromImage round-trip", () => { it("preserves a single file", async () => { @@ -86,8 +117,12 @@ describe("VFS image save/restore", () => { const image = await mfs.saveImage(); const restored = MemoryFileSystem.fromImage(image); - expect(readFile(restored, "/usr/local/bin/tool")).toEqual(new Uint8Array([1, 2, 3])); - expect(readFile(restored, "/usr/local/lib/libfoo.so")).toEqual(new Uint8Array([4, 5, 6])); + expect(readFile(restored, "/usr/local/bin/tool")).toEqual( + new Uint8Array([1, 2, 3]), + ); + expect(readFile(restored, "/usr/local/lib/libfoo.so")).toEqual( + new Uint8Array([4, 5, 6]), + ); expect(readFile(restored, "/etc/config")).toEqual(new Uint8Array([7, 8])); // Verify directory listing @@ -149,11 +184,15 @@ describe("VFS image save/restore", () => { // Can write new files writeFile(restored, "/new.txt", new TextEncoder().encode("new")); - expect(new TextDecoder().decode(readFile(restored, "/new.txt"))).toBe("new"); + expect(new TextDecoder().decode(readFile(restored, "/new.txt"))).toBe( + "new", + ); // Can modify existing files writeFile(restored, "/existing.txt", new TextEncoder().encode("updated")); - expect(new TextDecoder().decode(readFile(restored, "/existing.txt"))).toBe("updated"); + expect( + new TextDecoder().decode(readFile(restored, "/existing.txt")), + ).toBe("updated"); }); }); @@ -161,13 +200,20 @@ describe("VFS image save/restore", () => { it("preserves lazy file metadata by default", async () => { const mfs = createMemfs(); writeFile(mfs, "/real.txt", new TextEncoder().encode("real content")); - mfs.registerLazyFile("/bin/lazy-tool", "http://example.com/tool.wasm", 12345, 0o755); + mfs.registerLazyFile( + "/bin/lazy-tool", + "http://example.com/tool.wasm", + 12345, + 0o755, + ); const image = await mfs.saveImage(); const restored = MemoryFileSystem.fromImage(image); // Real file content preserved - expect(new TextDecoder().decode(readFile(restored, "/real.txt"))).toBe("real content"); + expect(new TextDecoder().decode(readFile(restored, "/real.txt"))).toBe( + "real content", + ); // Lazy file metadata preserved — stat reports declared size const st = restored.stat("/bin/lazy-tool"); @@ -182,6 +228,27 @@ describe("VFS image save/restore", () => { expect(entries[0].size).toBe(12345); }); + it("restores identity-less lazy metadata only from its coherent legacy image", async () => { + const mfs = createMemfs(); + mfs.registerLazyFile( + "/bin/legacy-tool", + "http://example.com/legacy.wasm", + 54321, + 0o755, + ); + + const restored = MemoryFileSystem.fromImage( + stripStandaloneLazyIdentity(await mfs.saveImage()), + ); + + expect(restored.getLazyEntry("/bin/legacy-tool")).toMatchObject({ + path: "/bin/legacy-tool", + url: "http://example.com/legacy.wasm", + size: 54321, + }); + expect(restored.stat("/bin/legacy-tool").size).toBe(54321); + }); + it("preserves multiple lazy files", async () => { const mfs = createMemfs(); mfs.registerLazyFile("/bin/a", "http://example.com/a.wasm", 100); @@ -200,29 +267,43 @@ describe("VFS image save/restore", () => { ]); }); - it("materializeAll clears lazy entries from image", async () => { + it("reconciles a peer-renamed lazy path before saving", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const fsA = MemoryFileSystem.create(sab); + const fsB = MemoryFileSystem.fromExisting(sab); + const url = "http://example.com/lazy.wasm"; + + fsA.registerLazyFile("/lazy", url, 12_345, 0o755); + fsB.rename("/lazy", "/moved"); + + const restored = MemoryFileSystem.fromImage(await fsA.saveImage()); + expect(() => restored.stat("/lazy")).toThrow(/No such file/); + expect(restored.stat("/moved").size).toBe(12_345); + expect(restored.getLazyEntry("/moved")).toMatchObject({ + path: "/moved", + url, + size: 12_345, + }); + expect(restored.exportLazyEntries()).toHaveLength(1); + }); + + it("explicit content replacement clears lazy entries from the image", async () => { const mfs = createMemfs(); - // Register a lazy file and manually write content to simulate materialization - // (can't actually fetch in tests, so we simulate by writing content before save) - mfs.registerLazyFile("/bin/tool", "http://example.com/tool.wasm", 5000, 0o755); + mfs.registerLazyFile( + "/bin/tool", + "http://example.com/tool.wasm", + 5000, + 0o755, + ); - // Manually write content to the file (simulating what ensureMaterialized does) const toolContent = new Uint8Array(100); for (let i = 0; i < 100; i++) toolContent[i] = i; const fd = mfs.open("/bin/tool", O_WRONLY | O_CREAT | O_TRUNC, 0o755); mfs.write(fd, toolContent, null, toolContent.length); mfs.close(fd); - // Clear the lazy entry manually to simulate materialization - // (ensureMaterialized would do this via fetch + delete) - const entries = mfs.exportLazyEntries(); - expect(entries).toHaveLength(1); - - // Instead, test that a save without lazy entries works correctly - // by creating a fresh mfs with no lazy files - const mfs2 = createMemfs(); - writeFile(mfs2, "/bin/tool", toolContent, 0o755); - const image = await mfs2.saveImage(); + expect(mfs.exportLazyEntries()).toHaveLength(0); + const image = await mfs.saveImage(); const restored = MemoryFileSystem.fromImage(image); expect(restored.exportLazyEntries()).toHaveLength(0); @@ -234,7 +315,11 @@ describe("VFS image save/restore", () => { writeFile(mfs, "/test.txt", new TextEncoder().encode("test")); const image = await mfs.saveImage(); - const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const view = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); const flags = view.getUint32(8, true); expect(flags & 1).toBe(0); // no lazy flag }); @@ -244,7 +329,11 @@ describe("VFS image save/restore", () => { mfs.registerLazyFile("/bin/tool", "http://example.com/tool.wasm", 1000); const image = await mfs.saveImage(); - const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const view = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); const flags = view.getUint32(8, true); expect(flags & 1).toBe(1); // lazy flag set }); @@ -261,7 +350,11 @@ describe("VFS image save/restore", () => { }); const image = await mfs.saveImage(); - const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const view = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); const flags = view.getUint32(8, true); expect(flags & 4).toBe(4); // metadata flag set @@ -277,8 +370,9 @@ describe("VFS image save/restore", () => { kernelAbi: 11, createdBy: "vfs-image.test", }); - expect(new TextDecoder().decode(readFile(restored, "/bin/tool"))) - .toBe("\0asm"); + expect(new TextDecoder().decode(readFile(restored, "/bin/tool"))).toBe( + "\0asm", + ); }); it("preserves metadata when a restored image is saved again", async () => { @@ -304,7 +398,11 @@ describe("VFS image save/restore", () => { const restored = MemoryFileSystem.fromImage(image); const cleared = await restored.saveImage({ metadata: null }); - const view = new DataView(cleared.buffer, cleared.byteOffset, cleared.byteLength); + const view = new DataView( + cleared.buffer, + cleared.byteOffset, + cleared.byteLength, + ); const flags = view.getUint32(8, true); expect(flags & 4).toBe(0); expect(MemoryFileSystem.readImageMetadata(cleared)).toBeNull(); @@ -322,10 +420,12 @@ describe("VFS image save/restore", () => { version: 1, kernelAbi: ABI_VERSION, }); - expect(MemoryFileSystem.fromImage(compressed).getImageMetadata()).toEqual({ - version: 1, - kernelAbi: ABI_VERSION, - }); + expect(MemoryFileSystem.fromImage(compressed).getImageMetadata()).toEqual( + { + version: 1, + kernelAbi: ABI_VERSION, + }, + ); }); it("rejects malformed metadata declarations", async () => { @@ -345,11 +445,16 @@ describe("VFS image save/restore", () => { metadata: { version: 1, kernelAbi: olderAbi }, }); - expect(() => MemoryFileSystem.assertImageKernelAbi(image, olderAbi)).not.toThrow(); - expect(() => MemoryFileSystem.assertImageKernelAbi(image, ABI_VERSION, "test image")) - .toThrow(new RegExp( + expect(() => + MemoryFileSystem.assertImageKernelAbi(image, olderAbi), + ).not.toThrow(); + expect(() => + MemoryFileSystem.assertImageKernelAbi(image, ABI_VERSION, "test image"), + ).toThrow( + new RegExp( `test image requires kernel ABI ${olderAbi}.*running kernel is ABI ${ABI_VERSION}`, - )); + ), + ); }); }); @@ -369,20 +474,28 @@ describe("VFS image save/restore", () => { expect(compressed[3]).toBe(0xfd); const restored = MemoryFileSystem.fromImage(compressed); - expect(new TextDecoder().decode(readFile(restored, "/hello.txt"))).toBe("compressed!"); - expect(readFile(restored, "/etc/config")).toEqual(new Uint8Array([1, 2, 3, 4])); + expect(new TextDecoder().decode(readFile(restored, "/hello.txt"))).toBe( + "compressed!", + ); + expect(readFile(restored, "/etc/config")).toEqual( + new Uint8Array([1, 2, 3, 4]), + ); }); it("fromImage with maxByteLength still works on zstd-wrapped image", async () => { const mfs = createMemfs(); writeFile(mfs, "/data.txt", new TextEncoder().encode("hi")); - const compressed = new Uint8Array(zstdCompressSync(await mfs.saveImage())); + const compressed = new Uint8Array( + zstdCompressSync(await mfs.saveImage()), + ); const restored = MemoryFileSystem.fromImage(compressed, { maxByteLength: 16 * 1024 * 1024, }); expect(restored.sharedBuffer.maxByteLength).toBe(16 * 1024 * 1024); - expect(new TextDecoder().decode(readFile(restored, "/data.txt"))).toBe("hi"); + expect(new TextDecoder().decode(readFile(restored, "/data.txt"))).toBe( + "hi", + ); }); it("saveImage writes a .vfs.zst that fromImage can restore", async () => { @@ -411,8 +524,9 @@ describe("VFS image save/restore", () => { const restored = MemoryFileSystem.fromImage(onDisk, { maxByteLength: 8 * 1024 * 1024, }); - expect(new TextDecoder().decode(readFile(restored, "/etc/hostname"))) - .toBe("wasmbox\n"); + expect( + new TextDecoder().decode(readFile(restored, "/etc/hostname")), + ).toBe("wasmbox\n"); const restoredBig = readFile(restored, "/usr/lib/libfoo.so"); expect(restoredBig.length).toBe(big.length); expect(restoredBig).toEqual(big); @@ -438,7 +552,9 @@ describe("VFS image save/restore", () => { it("rejects image with bad magic", () => { const bad = new Uint8Array(32); new DataView(bad.buffer).setUint32(0, 0xdeadbeef, true); - expect(() => MemoryFileSystem.fromImage(bad)).toThrow("Bad VFS image magic"); + expect(() => MemoryFileSystem.fromImage(bad)).toThrow( + "Bad VFS image magic", + ); }); it("rejects image with wrong version", () => { @@ -446,7 +562,9 @@ describe("VFS image save/restore", () => { const view = new DataView(bad.buffer); view.setUint32(0, 0x56465349, true); // VFSI magic view.setUint32(4, 99, true); // bad version - expect(() => MemoryFileSystem.fromImage(bad)).toThrow("Unsupported VFS image version"); + expect(() => MemoryFileSystem.fromImage(bad)).toThrow( + "Unsupported VFS image version", + ); }); it("rejects truncated image", () => { @@ -460,13 +578,19 @@ describe("VFS image save/restore", () => { }); it("rejects image that is too small", () => { - expect(() => MemoryFileSystem.fromImage(new Uint8Array(4))).toThrow("too small"); + expect(() => MemoryFileSystem.fromImage(new Uint8Array(4))).toThrow( + "too small", + ); }); it("image has correct magic and version", async () => { const mfs = createMemfs(); const image = await mfs.saveImage(); - const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const view = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); expect(view.getUint32(0, true)).toBe(0x56465349); // "VFSI" expect(view.getUint32(4, true)).toBe(1); // version 1 }); @@ -504,7 +628,9 @@ describe("VFS image save/restore", () => { const image = await mfs.saveImage(); const maxBytes = 16 * 1024 * 1024; - const restored = MemoryFileSystem.fromImage(image, { maxByteLength: maxBytes }); + const restored = MemoryFileSystem.fromImage(image, { + maxByteLength: maxBytes, + }); const buf = restored.sharedBuffer; expect(buf).toBeInstanceOf(SharedArrayBuffer); // The SAB should have maxByteLength set (growable) @@ -558,12 +684,18 @@ describe("VFS image save/restore", () => { writeFile(mfs, "/original.txt", new TextEncoder().encode("data")); const image = await mfs.saveImage(); - const restored = MemoryFileSystem.fromImage(image, { maxByteLength: 32 * 1024 * 1024 }); + const restored = MemoryFileSystem.fromImage(image, { + maxByteLength: 32 * 1024 * 1024, + }); // Can read existing files - expect(new TextDecoder().decode(readFile(restored, "/original.txt"))).toBe("data"); + expect( + new TextDecoder().decode(readFile(restored, "/original.txt")), + ).toBe("data"); // Can write new files writeFile(restored, "/new.txt", new TextEncoder().encode("new data")); - expect(new TextDecoder().decode(readFile(restored, "/new.txt"))).toBe("new data"); + expect(new TextDecoder().decode(readFile(restored, "/new.txt"))).toBe( + "new data", + ); }); it("without maxByteLength creates a non-growable SAB", async () => { @@ -588,7 +720,9 @@ describe("VFS image save/restore", () => { expect(rebased.sharedBuffer.byteLength).toBeGreaterThan(16 * 1024 * 1024); expect(rebased.sharedBuffer.maxByteLength).toBe(maxBytes); expect(stats.blocks * stats.bsize).toBe(maxBytes); - expect(new TextDecoder().decode(readFile(rebased, "/data.txt"))).toBe("base"); + expect(new TextDecoder().decode(readFile(rebased, "/data.txt"))).toBe( + "base", + ); }); it("raises the filesystem max beyond the source image superblock cap", async () => { @@ -605,30 +739,43 @@ describe("VFS image save/restore", () => { const restored = MemoryFileSystem.fromImage(image, { maxByteLength: rebaseMaxBytes, }); - expect(restored.statfs("/").blocks * restored.statfs("/").bsize).toBe(imageMaxBytes); + expect(restored.statfs("/").blocks * restored.statfs("/").bsize).toBe( + imageMaxBytes, + ); const rebased = restored.rebaseToNewFileSystem(rebaseMaxBytes); const stats = rebased.statfs("/"); expect(rebased.sharedBuffer.maxByteLength).toBe(rebaseMaxBytes); expect(stats.blocks * stats.bsize).toBe(rebaseMaxBytes); - expect(new TextDecoder().decode(readFile(rebased, "/data.txt"))).toBe("base"); + expect(new TextDecoder().decode(readFile(rebased, "/data.txt"))).toBe( + "base", + ); const rebasedImage = await rebased.saveImage(); const rerestored = MemoryFileSystem.fromImage(rebasedImage, { maxByteLength: rebaseMaxBytes, }); const rerestoredStats = rerestored.statfs("/"); - expect(rerestoredStats.blocks * rerestoredStats.bsize).toBe(rebaseMaxBytes); + expect(rerestoredStats.blocks * rerestoredStats.bsize).toBe( + rebaseMaxBytes, + ); }); it("preserves lazy file metadata without materializing stubs", async () => { const mfs = createMemfs(); writeFile(mfs, "/real.txt", new TextEncoder().encode("real")); - mfs.registerLazyFile("/bin/lazy-tool", "http://example.com/tool.wasm", 5_000_000, 0o755); + mfs.registerLazyFile( + "/bin/lazy-tool", + "http://example.com/tool.wasm", + 5_000_000, + 0o755, + ); const rebased = mfs.rebaseToNewFileSystem(16 * 1024 * 1024); - expect(new TextDecoder().decode(readFile(rebased, "/real.txt"))).toBe("real"); + expect(new TextDecoder().decode(readFile(rebased, "/real.txt"))).toBe( + "real", + ); expect(rebased.stat("/bin/lazy-tool").size).toBe(5_000_000); expect(rebased.stat("/bin/lazy-tool").mode & 0o777).toBe(0o755); expect(rebased.exportLazyEntries()).toMatchObject([ @@ -658,7 +805,9 @@ describe("VFS image save/restore", () => { writeFile(mfs, "/test.txt", new TextEncoder().encode("modified")); // Restored should still have original content - expect(new TextDecoder().decode(readFile(restored, "/test.txt"))).toBe("original"); + expect(new TextDecoder().decode(readFile(restored, "/test.txt"))).toBe( + "original", + ); }); it("modifications to restored filesystem don't affect original", async () => { @@ -673,7 +822,9 @@ describe("VFS image save/restore", () => { writeFile(restored, "/new.txt", new TextEncoder().encode("new")); // Original should be untouched - expect(new TextDecoder().decode(readFile(mfs, "/test.txt"))).toBe("original"); + expect(new TextDecoder().decode(readFile(mfs, "/test.txt"))).toBe( + "original", + ); expect(() => mfs.stat("/new.txt")).toThrow(); }); @@ -690,7 +841,9 @@ describe("VFS image save/restore", () => { expect(new TextDecoder().decode(readFile(r1, "/data.txt"))).toBe("r1"); expect(new TextDecoder().decode(readFile(r2, "/data.txt"))).toBe("r2"); - expect(new TextDecoder().decode(readFile(mfs, "/data.txt"))).toBe("shared"); + expect(new TextDecoder().decode(readFile(mfs, "/data.txt"))).toBe( + "shared", + ); }); }); }); diff --git a/host/test/vfs.test.ts b/host/test/vfs.test.ts index c257b3d60..aede93eb9 100644 --- a/host/test/vfs.test.ts +++ b/host/test/vfs.test.ts @@ -1,5 +1,11 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync, readFileSync, rmSync } from "node:fs"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { + mkdirSync, + mkdtempSync, + writeFileSync, + readFileSync, + rmSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { VirtualPlatformIO } from "../src/vfs/vfs"; @@ -397,13 +403,24 @@ describe("VirtualPlatformIO cross-mount rename (EXDEV)", () => { describe("HostFileSystem path traversal", () => { it("rejects paths that escape rootPath", () => { - const hfs = new HostFileSystem("/tmp/sandbox"); - expect(() => hfs.stat("/../../../etc/passwd")).toThrow("EACCES"); + const root = mkdtempSync(join(tmpdir(), "kandelo-host-fs-traversal-")); + try { + const hfs = new HostFileSystem(root); + expect(() => hfs.stat("/../../../etc/passwd")).toThrow("EACCES"); + } finally { + rmSync(root, { recursive: true, force: true }); + } }); it("rejects paths with embedded .. sequences", () => { - const hfs = new HostFileSystem("/tmp/sandbox"); - expect(() => hfs.stat("/subdir/../../etc/passwd")).toThrow("EACCES"); + const root = mkdtempSync(join(tmpdir(), "kandelo-host-fs-traversal-")); + try { + mkdirSync(join(root, "subdir")); + const hfs = new HostFileSystem(root); + expect(() => hfs.stat("/subdir/../../etc/passwd")).toThrow("EACCES"); + } finally { + rmSync(root, { recursive: true, force: true }); + } }); }); @@ -509,6 +526,90 @@ describe("MemoryFileSystem", () => { expect(entries).toContain("file.txt"); }); + it("reports raw inode numbers that remain representable after inode reuse", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_RDWR = 0x0002, + O_TRUNC = 0x0200; + + // SharedFS tracks an internal generation counter for reused inode slots. + // POSIX st_ino does not need to include that generation, and exposing it + // can overflow 32-bit guest language APIs while tools like ls(1) print the + // full kernel value. + for (let i = 0; i < 2_100; i++) { + const fd = mfs.open("/reuse.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + mfs.close(fd); + mfs.unlink("/reuse.txt"); + } + + const fd = mfs.open("/reuse.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const stat = mfs.fstat(fd); + expect(stat.ino).toBeGreaterThan(0); + expect(stat.ino).toBeLessThanOrEqual(0x7fffffff); + + const dh = mfs.opendir("/"); + let entry; + let dirIno: number | null = null; + while ((entry = mfs.readdir(dh)) !== null) { + if (entry.name === "reuse.txt") { + dirIno = entry.ino; + break; + } + } + mfs.closedir(dh); + expect(dirIno).toBe(stat.ino); + mfs.close(fd); + }); + + it("keeps large-directory indexes coherent across SharedFS instances", () => { + const sab = new SharedArrayBuffer(8 * 1024 * 1024); + const first = MemoryFileSystem.create(sab); + const second = MemoryFileSystem.fromExisting(sab); + first.mkdir("/bulk", 0o755); + + const names: string[] = []; + for (let i = 0; i < 340; i++) { + const name = `/bulk/${String(i).padStart(4, "0")}-${"x".repeat(180)}`; + names.push(name); + const fd = first.open(name, O_CREAT | O_RDWR, 0o644); + first.close(fd); + } + + // Populate the first mount's index, then reuse a deleted slot through a + // second mount without changing the directory's byte size. + expect(first.stat(names.at(-1)!).mode & 0xf000).toBe(0x8000); + second.unlink(names[100]); + const replacement = `/bulk/repl-${"y".repeat(180)}`; + const replacementFd = second.open(replacement, O_CREAT | O_RDWR, 0o644); + second.close(replacementFd); + + expect(first.stat(replacement).mode & 0xf000).toBe(0x8000); + expect(() => first.stat(names[100])).toThrow(/No such file/); + }); + + it("honors O_CREAT|O_EXCL by failing when the final path already exists", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_WRONLY = 0x0001, + O_CREAT = 0x0040, + O_EXCL = 0x0080; + + const fd = mfs.open("/exclusive.txt", O_WRONLY | O_CREAT | O_EXCL, 0o600); + mfs.close(fd); + + expect(() => + mfs.open("/exclusive.txt", O_WRONLY | O_CREAT | O_EXCL, 0o600), + ).toThrow(/File exists/); + + // POSIX open(O_CREAT|O_EXCL) must fail with EEXIST when the final path is + // a symbolic link, even if the symlink points at an existing regular file. + mfs.symlink("/exclusive.txt", "/exclusive-link.txt"); + expect(() => + mfs.open("/exclusive-link.txt", O_WRONLY | O_CREAT | O_EXCL, 0o600), + ).toThrow(/File exists/); + }); + it("stat returns correct size after writing", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); @@ -523,6 +624,37 @@ describe("MemoryFileSystem", () => { mfs.close(fd); }); + it("updates mtime and ctime after file writes and truncates", () => { + const now = vi.spyOn(Date, "now"); + try { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + now.mockReturnValue(1_000); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_RDWR = 0x0002, + O_TRUNC = 0x0200; + const fd = mfs.open("/timestamps.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const initial = mfs.fstat(fd); + + now.mockReturnValue(5_000); + mfs.write(fd, new TextEncoder().encode("abc"), null, 3); + const afterWrite = mfs.fstat(fd); + expect(afterWrite.mtimeMs).toBe(5_000); + expect(afterWrite.ctimeMs).toBe(5_000); + expect(afterWrite.mtimeMs).toBeGreaterThan(initial.mtimeMs); + + now.mockReturnValue(9_000); + mfs.ftruncate(fd, 1); + const afterTruncate = mfs.fstat(fd); + expect(afterTruncate.mtimeMs).toBe(9_000); + expect(afterTruncate.ctimeMs).toBe(9_000); + expect(afterTruncate.mtimeMs).toBeGreaterThan(afterWrite.mtimeMs); + mfs.close(fd); + } finally { + now.mockRestore(); + } + }); + it("unlink removes a file", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); @@ -534,6 +666,140 @@ describe("MemoryFileSystem", () => { expect(() => mfs.stat("/todelete.txt")).toThrow(); }); + it("rejects unlink paths with a trailing slash on non-directories", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_WRONLY = 0x0001; + + const fd = mfs.open("/file.txt", O_CREAT | O_WRONLY, 0o644); + mfs.close(fd); + mfs.symlink("/file.txt", "/link.txt"); + + expect(() => mfs.unlink("/file.txt/")).toThrow(/Not a directory/); + expect(() => mfs.unlink("/link.txt/")).toThrow(/Not a directory/); + expect(mfs.stat("/file.txt").mode & 0xf000).toBe(0x8000); + expect(mfs.readlink("/link.txt")).toBe("/file.txt"); + }); + + it("rejects rename source paths that require a non-directory to be a directory", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_WRONLY = 0x0001; + + const fd = mfs.open("/file.txt", O_CREAT | O_WRONLY, 0o644); + mfs.close(fd); + + expect(() => mfs.rename("/file.txt/", "/renamed.txt")).toThrow( + /Not a directory/, + ); + expect(mfs.stat("/file.txt").size).toBe(0); + expect(() => mfs.stat("/renamed.txt")).toThrow(/No such file/); + }); + + it("preserves POSIX type checks when renaming directories onto existing paths", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_WRONLY = 0x0001; + + mfs.mkdir("/dir", 0o755); + const fd = mfs.open("/file.txt", O_CREAT | O_WRONLY, 0o644); + mfs.close(fd); + mfs.symlink("/file.txt", "/link.txt"); + + expect(() => mfs.rename("/dir", "/file.txt")).toThrow(/Not a directory/); + expect(() => mfs.rename("/dir", "/link.txt")).toThrow(/Not a directory/); + + expect(mfs.stat("/dir").mode & 0xf000).toBe(0x4000); + expect(mfs.stat("/file.txt").mode & 0xf000).toBe(0x8000); + expect(mfs.readlink("/link.txt")).toBe("/file.txt"); + }); + + it("renames directories over empty directories and updates dot-dot", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_WRONLY = 0x0001; + + mfs.mkdir("/old-parent", 0o755); + mfs.mkdir("/new-parent", 0o755); + mfs.mkdir("/old-parent/child", 0o755); + const siblingFd = mfs.open( + "/new-parent/sibling.txt", + O_CREAT | O_WRONLY, + 0o644, + ); + mfs.close(siblingFd); + + mfs.rename("/old-parent/child", "/new-parent/child"); + expect(mfs.stat("/new-parent/child/../sibling.txt").mode & 0xf000).toBe( + 0x8000, + ); + + mfs.mkdir("/empty-dest", 0o755); + mfs.rename("/new-parent/child", "/empty-dest"); + expect(mfs.stat("/empty-dest").mode & 0xf000).toBe(0x4000); + expect(() => mfs.stat("/new-parent/child")).toThrow(/No such file/); + }); + + it("rejects rename and rmdir operands ending in dot or dot-dot", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + mfs.mkdir("/dir", 0o755); + mfs.mkdir("/dir/child", 0o755); + + expect(() => mfs.rename("/dir/.", "/moved")).toThrow(/Invalid argument/); + expect(() => mfs.rename("/dir/child", "/dir/..")).toThrow(/Invalid argument/); + expect(() => mfs.rmdir("/dir/.")).toThrow(/Invalid argument/); + expect(() => mfs.rmdir("/dir/child/..")).toThrow(/Invalid argument/); + expect(mfs.stat("/dir/child").mode & 0xf000).toBe(0x4000); + }); + + it("chmod and fchmod preserve the inode file type", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const fd = mfs.open("/regular", O_CREAT | O_RDWR, 0o644); + + mfs.chmod("/regular", 0o040755); + expect(mfs.stat("/regular").mode & 0xf000).toBe(0x8000); + mfs.fchmod(fd, 0o040700); + expect(mfs.fstat(fd).mode & 0xf000).toBe(0x8000); + mfs.close(fd); + }); + + it("keeps an unlinked open file alive until close", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_RDWR = 0x0002, + O_TRUNC = 0x0200; + + const oldFd = mfs.open("/open.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const oldData = new TextEncoder().encode("old"); + mfs.write(oldFd, oldData, null, oldData.length); + mfs.unlink("/open.txt"); + expect(() => mfs.stat("/open.txt")).toThrow(); + + const newFd = mfs.open("/open.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const newData = new TextEncoder().encode("newer"); + mfs.write(newFd, newData, null, newData.length); + + mfs.seek(oldFd, 0, 0); + const oldBuf = new Uint8Array(8); + const oldRead = mfs.read(oldFd, oldBuf, null, oldBuf.length); + expect(new TextDecoder().decode(oldBuf.subarray(0, oldRead))).toBe("old"); + + mfs.seek(newFd, 0, 0); + const newBuf = new Uint8Array(8); + const newRead = mfs.read(newFd, newBuf, null, newBuf.length); + expect(new TextDecoder().decode(newBuf.subarray(0, newRead))).toBe("newer"); + + mfs.close(oldFd); + mfs.close(newFd); + }); + it("ftruncate changes file size", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); @@ -735,4 +1001,14 @@ describe("NodeTimeProvider", () => { const ns2 = BigInt(t2.sec) * 1_000_000_000n + BigInt(t2.nsec); expect(ns2).toBeGreaterThanOrEqual(ns1); }); + + it("treats CLOCK_BOOTTIME as monotonic-equivalent", () => { + const tp = new NodeTimeProvider(); + const monotonic = tp.clockGettime(1); + const boottime = tp.clockGettime(7); + const monotonicNs = BigInt(monotonic.sec) * 1_000_000_000n + BigInt(monotonic.nsec); + const boottimeNs = BigInt(boottime.sec) * 1_000_000_000n + BigInt(boottime.nsec); + expect(boottimeNs).toBeGreaterThanOrEqual(monotonicNs); + expect(boottimeNs - monotonicNs).toBeLessThan(100_000_000n); + }); }); diff --git a/host/test/vfs/host-fs-path-resolution.test.ts b/host/test/vfs/host-fs-path-resolution.test.ts new file mode 100644 index 000000000..2da92fc60 --- /dev/null +++ b/host/test/vfs/host-fs-path-resolution.test.ts @@ -0,0 +1,157 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + existsSync, + mkdirSync, + mkdtempSync, + renameSync, + rmSync, + statSync, + symlinkSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { HostFileSystem } from "../../src/vfs/host-fs"; + +const O_WRONLY = 0o1; +const O_CREAT = 0o100; +const O_EXCL = 0o200; +const UTIME_OMIT = 0x3ffffffe; + +describe("HostFileSystem component-wise path resolution", () => { + let top: string; + let root: string; + let outside: string; + let hostFs: HostFileSystem; + + beforeEach(() => { + top = mkdtempSync(join(tmpdir(), "kandelo-host-fs-path-")); + root = join(top, "root"); + outside = join(top, "outside"); + mkdirSync(root); + mkdirSync(outside); + hostFs = new HostFileSystem(root, "/mnt"); + }); + + afterEach(() => { + rmSync(top, { recursive: true, force: true }); + }); + + it("allows ordinary names beginning with two dots and resolves components in order", () => { + writeFileSync(join(root, "..visible"), "visible"); + mkdirSync(join(root, "existing")); + writeFileSync(join(root, "existing", "file"), "data"); + + expect(hostFs.stat("/..visible").size).toBe(7); + expect(() => hostFs.stat("/existing/missing/../file")).toThrow(/ENOENT/); + }); + + it("follows normal relative and in-mount absolute symlinks", () => { + writeFileSync(join(root, "target.txt"), "target"); + symlinkSync("target.txt", join(root, "relative-link")); + symlinkSync("/mnt/target.txt", join(root, "absolute-link")); + + expect(hostFs.stat("/relative-link").size).toBe(6); + expect(hostFs.stat("/absolute-link").size).toBe(6); + expect(hostFs.readlink("/relative-link")).toBe("target.txt"); + + hostFs.unlink("/relative-link"); + expect(existsSync(join(root, "relative-link"))).toBe(false); + expect(existsSync(join(root, "target.txt"))).toBe(true); + }); + + it("does not follow a final symlink for an exclusive create", () => { + symlinkSync("created-through-link", join(root, "exclusive-link")); + + expect(() => + hostFs.open("/exclusive-link", O_WRONLY | O_CREAT | O_EXCL, 0o600), + ).toThrow(/EEXIST/); + expect(existsSync(join(root, "created-through-link"))).toBe(false); + }); + + it("does not follow a dangling final symlink for mkdir", () => { + symlinkSync("created-directory", join(root, "directory-link")); + + expect(() => hostFs.mkdir("/directory-link", 0o755)).toThrow(/EEXIST/); + expect(existsSync(join(root, "created-directory"))).toBe(false); + }); + + it("does not follow a dangling hard-link destination", () => { + writeFileSync(join(root, "source"), "source"); + symlinkSync("created-hard-link", join(root, "destination-link")); + + expect(() => hostFs.link("/source", "/destination-link")).toThrow(/EEXIST/); + expect(existsSync(join(root, "created-hard-link"))).toBe(false); + }); + + it("keeps native link semantics for a symlink source inside the mount", () => { + writeFileSync(join(root, "target"), "target"); + symlinkSync("target", join(root, "source-link")); + + hostFs.link("/source-link", "/linked-symlink"); + + const sourceLink = hostFs.lstat("/source-link"); + const target = hostFs.stat("/target"); + const linked = hostFs.lstat("/linked-symlink"); + expect([sourceLink.ino, target.ino]).toContain(linked.ino); + if ((linked.mode & 0xf000) === 0xa000) { + expect(hostFs.readlink("/linked-symlink")).toBe("target"); + } else { + expect(linked.ino).toBe(target.ino); + } + }); + + it("revalidates an intermediate directory after an external replacement", () => { + mkdirSync(join(root, "cached")); + writeFileSync(join(root, "cached", "inside"), "inside"); + writeFileSync(join(outside, "secret"), "outside-secret"); + + expect(hostFs.stat("/cached/inside").size).toBe(6); + renameSync(join(root, "cached"), join(root, "old-cached")); + symlinkSync(outside, join(root, "cached")); + + expect(() => hostFs.stat("/cached/secret")).toThrow(/EACCES/); + }); +}); + +describe("HostFileSystem utimens metadata", () => { + let root: string; + let hostFs: HostFileSystem; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "kandelo-host-fs-utimens-")); + hostFs = new HostFileSystem(root); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it("preserves an omitted timestamp", () => { + const nativePath = join(root, "timestamps"); + writeFileSync(nativePath, "data"); + utimesSync(nativePath, 10, 20); + const before = statSync(nativePath); + + hostFs.utimensat("/timestamps", 0, UTIME_OMIT, 30, 0); + + const result = hostFs.stat("/timestamps"); + expect(result.atimeMs).toBe(before.atimeMs); + expect(result.mtimeMs).toBe(30_000); + }); + + it("drops timestamp overrides after an external native mutation", () => { + const nativePath = join(root, "externally-mutated"); + writeFileSync(nativePath, "before"); + hostFs.utimensat("/externally-mutated", 1, 0, 2, 0); + expect(hostFs.stat("/externally-mutated").mtimeMs).toBe(2_000); + + writeFileSync(nativePath, "after external mutation"); + const nativeAfter = statSync(nativePath); + + expect(hostFs.stat("/externally-mutated").mtimeMs).toBe( + nativeAfter.mtimeMs, + ); + }); +}); diff --git a/host/test/virtual-network.test.ts b/host/test/virtual-network.test.ts index dd0fae50b..44718adf8 100644 --- a/host/test/virtual-network.test.ts +++ b/host/test/virtual-network.test.ts @@ -10,6 +10,7 @@ const POLLIN = 0x0001; const POLLOUT = 0x0004; const POLLERR = 0x0008; const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; describe("LocalVirtualNetwork", () => { it("resolves bounded legacy numeric IPv4 forms and valid DNS aliases", () => { @@ -56,6 +57,27 @@ describe("LocalVirtualNetwork", () => { expect(new TextDecoder().decode(client.recv(7, 16, 0))).toBe("pong"); }); + it("honors MSG_PEEK without consuming TCP stream data", () => { + const net = new LocalVirtualNetwork(); + const server = net.attachMachine({ id: "server", address: [10, 88, 0, 2] }); + const client = net.attachMachine({ id: "client", address: [10, 88, 0, 3] }); + let accepted: TcpConnectionPeer | null = null; + + expect(server.listenTcp!("srv:1", new Uint8Array([10, 88, 0, 2]), 8080, { + accept(peer) { + accepted = peer; + return 0; + }, + })).toBe(0); + + client.connect(7, new Uint8Array([10, 88, 0, 2]), 8080); + expect(accepted).not.toBeNull(); + accepted!.send(new TextEncoder().encode("peek-data"), 0); + + expect(new TextDecoder().decode(client.recv(7, 4, MSG_PEEK))).toBe("peek"); + expect(new TextDecoder().decode(client.recv(7, 9, 0))).toBe("peek-data"); + }); + it("reports refused TCP connects when no listener is bound", () => { const net = new LocalVirtualNetwork(); const client = net.attachMachine({ id: "client", address: [10, 88, 0, 3] }); diff --git a/host/test/vm-interrupt-timer.test.ts b/host/test/vm-interrupt-timer.test.ts new file mode 100644 index 000000000..f5b78af10 --- /dev/null +++ b/host/test/vm-interrupt-timer.test.ts @@ -0,0 +1,251 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + MAX_VM_INTERRUPT_TIMER_DELAY_MS, + VmInterruptTimerManager, + type VmInterruptTimerScheduler, +} from "../src/vm-interrupt-timer"; + +interface Generation { + memory: WebAssembly.Memory; +} + +function generation(shared = true): Generation { + return { + memory: new WebAssembly.Memory({ + initial: 1, + maximum: 2, + shared, + }), + }; +} + +function flag(generation: Generation, ptr: number): number { + return Atomics.load(new Uint8Array(generation.memory.buffer), ptr); +} + +describe("VmInterruptTimerManager", () => { + let nowMs: number; + let current: Map; + let delays: number[]; + let scheduler: VmInterruptTimerScheduler; + let manager: VmInterruptTimerManager; + + beforeEach(() => { + vi.useFakeTimers(); + nowMs = 0; + current = new Map(); + delays = []; + scheduler = { + now: () => nowMs, + set: (callback, delayMs) => { + delays.push(delayMs); + return setTimeout(callback, delayMs); + }, + clear: (handle) => clearTimeout(handle), + }; + manager = new VmInterruptTimerManager((pid) => current.get(pid), scheduler); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + function advance(ms: number): void { + nowMs += ms; + vi.advanceTimersByTime(ms); + } + + it("sets both flags at the exact monotonic deadline", () => { + const process = generation(); + current.set(41, process); + + expect(manager.arm(41, process, { + timedOutPtr: 12, + vmInterruptPtr: 13, + seconds: 1, + })).toBe(true); + + advance(999); + expect(flag(process, 12)).toBe(0); + expect(flag(process, 13)).toBe(0); + + advance(1); + expect(flag(process, 12)).toBe(1); + expect(flag(process, 13)).toBe(1); + expect(manager.activeCount).toBe(0); + }); + + it("re-arms from the new request time and suppresses the old callback", () => { + const process = generation(); + current.set(42, process); + + manager.arm(42, process, { + timedOutPtr: 20, + vmInterruptPtr: 21, + seconds: 1, + }); + advance(500); + manager.arm(42, process, { + timedOutPtr: 20, + vmInterruptPtr: 21, + seconds: 2, + }); + + advance(500); + expect(flag(process, 20)).toBe(0); + advance(1_499); + expect(flag(process, 20)).toBe(0); + advance(1); + expect(flag(process, 20)).toBe(1); + }); + + it("cancels through a non-positive runtime-hook request", () => { + const process = generation(); + current.set(43, process); + manager.arm(43, process, { + timedOutPtr: 24, + vmInterruptPtr: 25, + seconds: 1, + }); + + expect(manager.handleRequest(43, process, { + timedOutPtr: 24, + vmInterruptPtr: 25, + seconds: 0, + })).toBe(true); + advance(1_000); + + expect(flag(process, 24)).toBe(0); + expect(flag(process, 25)).toBe(0); + expect(manager.activeCount).toBe(0); + }); + + it("refuses a stale process generation without disturbing its replacement", () => { + const oldProcess = generation(); + const newProcess = generation(); + current.set(44, oldProcess); + manager.arm(44, oldProcess, { + timedOutPtr: 28, + vmInterruptPtr: 29, + seconds: 1, + }); + + current.set(44, newProcess); + manager.arm(44, newProcess, { + timedOutPtr: 30, + vmInterruptPtr: 31, + seconds: 1, + }); + expect(manager.cancel(44, oldProcess)).toBe(false); + + advance(1_000); + expect(flag(oldProcess, 28)).toBe(0); + expect(flag(newProcess, 30)).toBe(1); + expect(flag(newProcess, 31)).toBe(1); + }); + + it("drops a queued callback when the PID generation changes", () => { + const oldProcess = generation(); + const replacement = generation(); + current.set(45, oldProcess); + manager.arm(45, oldProcess, { + timedOutPtr: 32, + vmInterruptPtr: 33, + seconds: 1, + }); + + current.set(45, replacement); + advance(1_000); + + expect(flag(oldProcess, 32)).toBe(0); + expect(flag(replacement, 32)).toBe(0); + expect(manager.activeCount).toBe(0); + }); + + it("chunks delays above the JavaScript signed-32-bit timer limit", () => { + const process = generation(); + current.set(46, process); + const tailMs = 2_500; + manager.arm(46, process, { + timedOutPtr: 36, + vmInterruptPtr: 37, + seconds: (MAX_VM_INTERRUPT_TIMER_DELAY_MS + tailMs) / 1_000, + }); + + expect(delays).toEqual([MAX_VM_INTERRUPT_TIMER_DELAY_MS]); + advance(MAX_VM_INTERRUPT_TIMER_DELAY_MS); + expect(flag(process, 36)).toBe(0); + expect(delays).toEqual([MAX_VM_INTERRUPT_TIMER_DELAY_MS, tailMs]); + + advance(tailMs - 1); + expect(flag(process, 36)).toBe(0); + advance(1); + expect(flag(process, 36)).toBe(1); + }); + + it("rejects out-of-bounds and non-shared flag storage", () => { + const process = generation(); + current.set(47, process); + expect(manager.arm(47, process, { + timedOutPtr: process.memory.buffer.byteLength, + vmInterruptPtr: 1, + seconds: 1, + })).toBe(false); + + const unshared = generation(false); + current.set(48, unshared); + expect(manager.arm(48, unshared, { + timedOutPtr: 1, + vmInterruptPtr: 2, + seconds: 1, + })).toBe(false); + expect(manager.activeCount).toBe(0); + }); + + it("clears every process timer", () => { + const first = generation(); + const second = generation(); + current.set(49, first); + current.set(50, second); + manager.arm(49, first, { timedOutPtr: 40, vmInterruptPtr: 41, seconds: 1 }); + manager.arm(50, second, { timedOutPtr: 42, vmInterruptPtr: 43, seconds: 1 }); + + manager.clearAll(); + advance(1_000); + + expect(flag(first, 40)).toBe(0); + expect(flag(second, 42)).toBe(0); + expect(manager.activeCount).toBe(0); + }); + + it("cancels a timer whose scheduler handle is numeric zero", () => { + const process = generation(); + const callbacks: Array<() => void> = []; + const cleared: number[] = []; + const zeroHandleScheduler: VmInterruptTimerScheduler = { + now: () => 0, + set: (callback) => { + callbacks.push(callback); + return 0; + }, + clear: (handle) => cleared.push(handle), + }; + current.set(51, process); + const zeroManager = new VmInterruptTimerManager( + (pid) => current.get(pid), + zeroHandleScheduler, + ); + + zeroManager.arm(51, process, { + timedOutPtr: 44, + vmInterruptPtr: 45, + seconds: 1, + }); + expect(zeroManager.cancel(51, process)).toBe(true); + expect(cleared).toEqual([0]); + + callbacks[0](); + expect(flag(process, 44)).toBe(0); + expect(flag(process, 45)).toBe(0); + }); +}); diff --git a/host/test/wasm-trap.test.ts b/host/test/wasm-trap.test.ts index f48a0a346..464450924 100644 --- a/host/test/wasm-trap.test.ts +++ b/host/test/wasm-trap.test.ts @@ -45,7 +45,7 @@ const programsBuilt = existsSync(wasmTrapBin) && describe.skipIf(!programsBuilt)("wasm trap → host exit (regression)", () => { it("__builtin_trap() in user code: spawn() resolves promptly, no hang", async () => { const t0 = Date.now(); - const { exitCode, stderr } = await runCentralizedProgram({ + const { exitCode, stderr, hostDiagnostics } = await runCentralizedProgram({ programPath: wasmTrapBin, useDefaultRootfs: false, timeout: 5000, @@ -54,17 +54,19 @@ describe.skipIf(!programsBuilt)("wasm trap → host exit (regression)", () => { expect(stderr).toContain("before-trap"); expect(stderr).not.toContain("SHOULD-NEVER-REACH"); + expect(stderr).not.toContain("RuntimeError"); // Arbitrary `unreachable` traps are no longer masked as successful exits; // only the known kernel_exit path is interpreted as normal termination. expect(exitCode).toBe(signalExitStatus(SIGILL)); - expect(stderr).toContain("RuntimeError"); - expect(stderr).toContain("unreachable"); + const diagnosticText = hostDiagnostics.map((entry) => entry.message).join("\n"); + expect(diagnosticText).toContain("RuntimeError"); + expect(diagnosticText).toContain("unreachable"); expect(elapsed).toBeLessThan(3000); }, 8_000); it("out-of-bounds memory trap resolves as SIGSEGV", async () => { const t0 = Date.now(); - const { exitCode, stderr } = await runCentralizedProgram({ + const { exitCode, stderr, hostDiagnostics } = await runCentralizedProgram({ programPath: oobTrapBin, useDefaultRootfs: false, timeout: 5000, @@ -73,14 +75,15 @@ describe.skipIf(!programsBuilt)("wasm trap → host exit (regression)", () => { expect(stderr).toContain("before-oob"); expect(stderr).not.toContain("SHOULD-NEVER-REACH"); + expect(stderr).not.toContain("RuntimeError"); expect(exitCode).toBe(signalExitStatus(SIGSEGV)); - expect(stderr).toContain("RuntimeError"); + expect(hostDiagnostics.map((entry) => entry.message).join("\n")).toContain("RuntimeError"); expect(elapsed).toBeLessThan(3000); }, 8_000); it("integer divide-by-zero trap resolves as SIGFPE", async () => { const t0 = Date.now(); - const { exitCode, stderr } = await runCentralizedProgram({ + const { exitCode, stderr, hostDiagnostics } = await runCentralizedProgram({ programPath: divzeroTrapBin, useDefaultRootfs: false, timeout: 5000, @@ -89,8 +92,9 @@ describe.skipIf(!programsBuilt)("wasm trap → host exit (regression)", () => { expect(stderr).toContain("before-divzero"); expect(stderr).not.toContain("SHOULD-NEVER-REACH"); + expect(stderr).not.toContain("RuntimeError"); expect(exitCode).toBe(signalExitStatus(SIGFPE)); - expect(stderr).toContain("RuntimeError"); + expect(hostDiagnostics.map((entry) => entry.message).join("\n")).toContain("RuntimeError"); expect(elapsed).toBeLessThan(3000); }, 8_000); diff --git a/host/test/worker-adapter.test.ts b/host/test/worker-adapter.test.ts index 767a62837..0df11a803 100644 --- a/host/test/worker-adapter.test.ts +++ b/host/test/worker-adapter.test.ts @@ -1,5 +1,11 @@ +import { fileURLToPath } from "node:url"; import { describe, it, expect } from "vitest"; -import { MockWorkerAdapter } from "../src/worker-adapter"; +import { + MockWorkerAdapter, + NodeWorkerAdapter, + nodeWorkerOptions, + nodeWorkerStackSizeMb, +} from "../src/worker-adapter"; describe("MockWorkerAdapter", () => { it("should create a worker handle and capture workerData", () => { @@ -48,3 +54,75 @@ describe("MockWorkerAdapter", () => { expect(adapter.lastWorker!.sentMessages).toEqual([{ type: "terminate" }]); }); }); + +describe("NodeWorkerAdapter stack policy", () => { + it("uses 32 MiB by default and validates explicit overrides", () => { + expect(nodeWorkerStackSizeMb(undefined)).toBe(32); + expect(nodeWorkerStackSizeMb("48")).toBe(48); + expect(() => nodeWorkerStackSizeMb("0")).toThrow(/invalid/); + expect(() => nodeWorkerStackSizeMb("-1")).toThrow(/invalid/); + expect(() => nodeWorkerStackSizeMb("not-a-number")).toThrow(/invalid/); + }); + + it("preserves other resource limits when setting the stack limit", () => { + expect(nodeWorkerOptions({ pid: 7 }, { + resourceLimits: { maxOldGenerationSizeMb: 64 }, + })).toMatchObject({ + workerData: { pid: 7 }, + resourceLimits: { + maxOldGenerationSizeMb: 64, + stackSizeMb: 32, + }, + }); + }); + + it("runs a deeply recursive Wasm workload inside the configured worker stack", async () => { + const adapter = new NodeWorkerAdapter( + new URL("./fixtures/deep-wasm-recursion-worker.mjs", import.meta.url), + ); + const worker = adapter.createWorker({ + wasmPath: fileURLToPath( + new URL("./fixtures/deep-wasm-recursion.wasm", import.meta.url), + ), + depth: 300_000, + }); + + try { + const result = await new Promise((resolve, reject) => { + worker.on("message", resolve); + worker.on("error", reject); + worker.on("exit", (code) => { + if (code !== 0) reject(new Error(`deep Wasm worker exited ${code}`)); + }); + }); + expect(result).toEqual({ result: 300_000 }); + } finally { + await worker.terminate(); + } + }, 20_000); + + it("supports the default four concurrent process-worker stack reservations", async () => { + const adapter = new NodeWorkerAdapter( + new URL("./fixtures/deep-wasm-recursion-worker.mjs", import.meta.url), + ); + const wasmPath = fileURLToPath( + new URL("./fixtures/deep-wasm-recursion.wasm", import.meta.url), + ); + const workers = Array.from({ length: 4 }, () => + adapter.createWorker({ wasmPath, depth: 100_000 })); + + try { + const results = await Promise.all(workers.map((worker) => + new Promise((resolve, reject) => { + worker.on("message", resolve); + worker.on("error", reject); + worker.on("exit", (code) => { + if (code !== 0) reject(new Error(`deep Wasm worker exited ${code}`)); + }); + }))); + expect(results).toEqual(Array.from({ length: 4 }, () => ({ result: 100_000 }))); + } finally { + await Promise.all(workers.map((worker) => worker.terminate())); + } + }, 20_000); +}); diff --git a/images/rootfs/etc/services b/images/rootfs/etc/services index 8ba68d7ef..bd91937a3 100644 --- a/images/rootfs/etc/services +++ b/images/rootfs/etc/services @@ -13,7 +13,7 @@ domain 53/tcp domain 53/udp gopher 70/tcp finger 79/tcp -http 80/tcp www +http 80/tcp www www-http pop3 110/tcp pop-3 nntp 119/tcp readnews untp ntp 123/udp @@ -22,3 +22,5 @@ snmp 161/udp https 443/tcp imaps 993/tcp pop3s 995/tcp +mysql 3306/tcp +postgresql 5432/tcp diff --git a/images/vfs/scripts/dinit-image-helpers.ts b/images/vfs/scripts/dinit-image-helpers.ts index d220ce033..ae10aedd8 100644 --- a/images/vfs/scripts/dinit-image-helpers.ts +++ b/images/vfs/scripts/dinit-image-helpers.ts @@ -146,12 +146,23 @@ const ETC_HOSTS = [ "", ].join("\n"); -const ETC_SERVICES = [ - "http\t\t80/tcp\t\twww", - "https\t\t443/tcp", - "mysql\t\t3306/tcp", - "", -].join("\n"); +const ETC_SERVICES = readFileSync( + join(REPO_ROOT, "images", "rootfs", "etc", "services"), + "utf8", +); + +/** + * Install the account and network databases shared by dinit-based images. + * `/etc/services` comes from the rootfs source so derived images cannot drift + * into a second, smaller service-name contract. + */ +export function addDinitBaseSystemFiles(fs: MemoryFileSystem): void { + ensureDirRecursive(fs, "/etc"); + writeVfsFile(fs, "/etc/passwd", ETC_PASSWD); + writeVfsFile(fs, "/etc/group", ETC_GROUP); + writeVfsFile(fs, "/etc/hosts", ETC_HOSTS); + writeVfsFile(fs, "/etc/services", ETC_SERVICES); +} /** * Options for {@link addDinitInit}. The defaults set up an implicit @@ -264,11 +275,7 @@ export function addDinitInit( // Basic rootfs files. Most Unix daemons expect these to exist at // startup; missing them is the usual cause of "started but exits 1 // silently" failures. - ensureDirRecursive(fs, "/etc"); - writeVfsFile(fs, "/etc/passwd", ETC_PASSWD); - writeVfsFile(fs, "/etc/group", ETC_GROUP); - writeVfsFile(fs, "/etc/hosts", ETC_HOSTS); - writeVfsFile(fs, "/etc/services", ETC_SERVICES); + addDinitBaseSystemFiles(fs); // Standard runtime/log dirs ensureDirRecursive(fs, "/var/log"); diff --git a/images/vfs/scripts/shell-vfs-build.ts b/images/vfs/scripts/shell-vfs-build.ts index 969c554eb..fb1087c42 100644 --- a/images/vfs/scripts/shell-vfs-build.ts +++ b/images/vfs/scripts/shell-vfs-build.ts @@ -162,34 +162,6 @@ function populateSystem(fs: MemoryFileSystem): void { fs.chown("/home/user", 1000, 1000); populateNetHackPlayground(fs); - // /etc/services — required for getservbyname/getservbyport calls in - // nginx/php-fpm/MariaDB. Harmless in Shell-only builds. - const services = [ - "tcpmux\t\t1/tcp", - "echo\t\t7/tcp", - "echo\t\t7/udp", - "discard\t\t9/tcp\t\tsink null", - "discard\t\t9/udp\t\tsink null", - "ftp-data\t20/tcp", - "ftp\t\t21/tcp", - "ssh\t\t22/tcp", - "telnet\t\t23/tcp", - "smtp\t\t25/tcp\t\tmail", - "domain\t\t53/tcp", - "domain\t\t53/udp", - "http\t\t80/tcp\t\twww", - "pop3\t\t110/tcp\t\tpop-3", - "nntp\t\t119/tcp\t\treadnews untp", - "ntp\t\t123/udp", - "imap\t\t143/tcp\t\timap2", - "snmp\t\t161/udp", - "https\t\t443/tcp", - "imaps\t\t993/tcp", - "pop3s\t\t995/tcp", - "mysql\t\t3306/tcp", - ].join("\n") + "\n"; - writeVfsFile(fs, "/etc/services", services); - const gitconfig = [ "[maintenance]", "\tauto = false", diff --git a/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c b/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c index f933623bc..6293689f6 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c @@ -1,11 +1,23 @@ +#define _GNU_SOURCE #include +#include +#include #include #include +#include "syscall.h" char *if_indextoname(unsigned index, char *name) { - if (index == 1) - return strncpy(name, "lo", IF_NAMESIZE); - errno = ENXIO; - return 0; + struct ifreq ifr = {0}; + int fd, r; + + if ((fd = socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) return 0; + ifr.ifr_ifindex = index; + r = ioctl(fd, SIOCGIFNAME, &ifr); + __syscall(SYS_close, fd); + if (r < 0) { + if (errno == ENODEV) errno = ENXIO; + return 0; + } + return strncpy(name, ifr.ifr_name, IF_NAMESIZE); } diff --git a/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c b/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c index 72ca21704..5ff17a8b9 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c @@ -1,19 +1,49 @@ +#define _GNU_SOURCE #include +#include +#include #include #include +#include "syscall.h" struct if_nameindex *if_nameindex(void) { - /* Return a synthetic loopback interface */ - struct if_nameindex *idx = malloc(2 * sizeof(*idx)); - if (!idx) return 0; - idx[0].if_index = 1; - idx[0].if_name = strdup("lo"); - if (!idx[0].if_name) { - free(idx); - return 0; + struct ifconf ifc = {0}; + struct ifreq *req = 0; + struct if_nameindex *idx = 0; + size_t count = 0; + int fd; + + if ((fd = socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) return 0; + + /* Query the host-visible interface list instead of embedding interface + * numbers in libc. The terminating zero entry comes from calloc. */ + if (ioctl(fd, SIOCGIFCONF, &ifc) < 0 || ifc.ifc_len < 0) goto fail; + if (ifc.ifc_len) { + req = malloc(ifc.ifc_len); + if (!req) goto fail; + ifc.ifc_req = req; + if (ioctl(fd, SIOCGIFCONF, &ifc) < 0) goto fail; + count = ifc.ifc_len / sizeof(*req); + } + + idx = calloc(count + 1, sizeof(*idx)); + if (!idx) goto fail; + + for (size_t i = 0; i < count; i++) { + if (ioctl(fd, SIOCGIFINDEX, &req[i]) < 0) goto fail; + idx[i].if_index = req[i].ifr_ifindex; + idx[i].if_name = strdup(req[i].ifr_name); + if (!idx[i].if_name) goto fail; } - idx[1].if_index = 0; - idx[1].if_name = 0; + __syscall(SYS_close, fd); + free(req); return idx; + +fail: + __syscall(SYS_close, fd); + for (size_t i = 0; i < count; i++) free(idx ? idx[i].if_name : 0); + free(idx); + free(req); + return 0; } diff --git a/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c b/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c index 4c837e63a..540da56d7 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c @@ -1,8 +1,18 @@ +#define _GNU_SOURCE #include +#include +#include #include +#include "syscall.h" unsigned if_nametoindex(const char *name) { - if (!strcmp(name, "lo")) return 1; - return 0; + struct ifreq ifr = {0}; + int fd, r; + + if ((fd = socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) return 0; + strncpy(ifr.ifr_name, name, sizeof ifr.ifr_name - 1); + r = ioctl(fd, SIOCGIFINDEX, &ifr); + __syscall(SYS_close, fd); + return r < 0 ? 0 : ifr.ifr_ifindex; } diff --git a/packages/registry/erlang-vfs/build.toml b/packages/registry/erlang-vfs/build.toml index 071017f82..2971c033f 100644 --- a/packages/registry/erlang-vfs/build.toml +++ b/packages/registry/erlang-vfs/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/erlang-vfs/build-erlang-vfs.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/kandelo-sdk/build.toml b/packages/registry/kandelo-sdk/build.toml index a993a28bc..a34738eea 100644 --- a/packages/registry/kandelo-sdk/build.toml +++ b/packages/registry/kandelo-sdk/build.toml @@ -26,7 +26,7 @@ inputs = [ ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/lamp/build.toml b/packages/registry/lamp/build.toml index cfa890205..3153c8086 100644 --- a/packages/registry/lamp/build.toml +++ b/packages/registry/lamp/build.toml @@ -8,6 +8,7 @@ inputs = [ "images/vfs/scripts/build-lamp-vfs-image.sh", "images/vfs/scripts/build-lamp-vfs-image.ts", "images/vfs/scripts/dinit-image-helpers.ts", + "images/rootfs/etc/services", "images/vfs/scripts/kandelo-demo-config.ts", "images/vfs/scripts/kandelo-demo-guides.ts", "images/vfs/scripts/opcache-prewarm.ts", @@ -31,7 +32,7 @@ inputs = [ ] repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 7 +revision = 8 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/lamp/package.toml b/packages/registry/lamp/package.toml index d898b8a55..b7ad6ee9c 100644 --- a/packages/registry/lamp/package.toml +++ b/packages/registry/lamp/package.toml @@ -14,7 +14,7 @@ depends_on = [ "shell@0.1.0", "mariadb@10.5.28", "nginx@1.24.0", - "php@8.3.2", + "php@8.3.15", "dinit@0.19.4", "msmtpd@1.8.32", ] diff --git a/packages/registry/mariadb-test/build.toml b/packages/registry/mariadb-test/build.toml index 8a6904e51..6a762ae93 100644 --- a/packages/registry/mariadb-test/build.toml +++ b/packages/registry/mariadb-test/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/mariadb-test/build-mariadb-test.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 1 +revision = 2 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/mariadb-vfs/build.toml b/packages/registry/mariadb-vfs/build.toml index 5a456a5ee..7e7be2a37 100644 --- a/packages/registry/mariadb-vfs/build.toml +++ b/packages/registry/mariadb-vfs/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/mariadb-vfs/build-mariadb-vfs.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/netcat/build-netcat.sh b/packages/registry/netcat/build-netcat.sh index 1ae238268..378791c15 100755 --- a/packages/registry/netcat/build-netcat.sh +++ b/packages/registry/netcat/build-netcat.sh @@ -3,15 +3,28 @@ set -euo pipefail # Build GNU Netcat 0.7.1 for wasm32-posix-kernel. -NETCAT_VERSION="${NETCAT_VERSION:-0.7.1}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -SRC_DIR="$SCRIPT_DIR/netcat-src" -BIN_DIR="$SCRIPT_DIR/bin" +NETCAT_VERSION="${WASM_POSIX_DEP_VERSION:-${NETCAT_VERSION:-0.7.1}}" +SOURCE_URL="${WASM_POSIX_DEP_SOURCE_URL:-https://downloads.sourceforge.net/project/netcat/netcat/${NETCAT_VERSION}/netcat-${NETCAT_VERSION}.tar.gz}" +SOURCE_SHA256="${WASM_POSIX_DEP_SOURCE_SHA256:-30719c9a4ffbcf15676b8f528233ccc54ee6cba96cb4590975f5fd60c68a066f}" +TARGET_ARCH="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" +WORK_DIR="${WASM_POSIX_DEP_WORK_DIR:-$SCRIPT_DIR}" +SRC_DIR="$WORK_DIR/netcat-src" +BIN_DIR="$WORK_DIR/bin" SYSROOT="${WASM_POSIX_SYSROOT:-$REPO_ROOT/sysroot}" +# Worktree-local SDK on PATH (no global npm link required). +# shellcheck source=/dev/null +source "$REPO_ROOT/sdk/activate.sh" + +if [ "$TARGET_ARCH" != "wasm32" ]; then + echo "ERROR: GNU Netcat is currently packaged for wasm32 only, got $TARGET_ARCH" >&2 + exit 2 +fi + if ! command -v wasm32posix-cc &>/dev/null; then - echo "ERROR: wasm32posix-cc not found. Run 'npm link' in sdk/ first." >&2 + echo "ERROR: wasm32posix-cc not found after sourcing sdk/activate.sh." >&2 exit 1 fi @@ -23,14 +36,26 @@ fi export WASM_POSIX_SYSROOT="$SYSROOT" export WASM_POSIX_GLUE_DIR="$REPO_ROOT/libc/glue" +SOURCE_MARKER="$SRC_DIR/.kandelo-netcat-source" +expected_source_marker="$(printf '%s\n%s\n%s' "$NETCAT_VERSION" "$SOURCE_URL" "$SOURCE_SHA256")" +if [ -d "$SRC_DIR" ] && [ "$(cat "$SOURCE_MARKER" 2>/dev/null || true)" != "$expected_source_marker" ]; then + echo "==> Existing GNU Netcat source does not match requested version/source; cleaning..." + rm -rf "$SRC_DIR" "$BIN_DIR" +fi + if [ ! -d "$SRC_DIR" ]; then echo "==> Downloading GNU Netcat $NETCAT_VERSION..." + DOWNLOAD_DIR="$(mktemp -d "${TMPDIR:-/tmp}/kandelo-netcat-src.XXXXXX")" + trap 'rm -rf "$DOWNLOAD_DIR"' EXIT TARBALL="netcat-${NETCAT_VERSION}.tar.gz" - URL="https://downloads.sourceforge.net/project/netcat/netcat/${NETCAT_VERSION}/${TARBALL}" - curl --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors -fsSL "$URL" -o "/tmp/$TARBALL" + curl --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors -fsSL "$SOURCE_URL" -o "$DOWNLOAD_DIR/$TARBALL" + echo "==> Verifying source sha256..." + echo "$SOURCE_SHA256 $DOWNLOAD_DIR/$TARBALL" | shasum -a 256 -c - mkdir -p "$SRC_DIR" - tar xzf "/tmp/$TARBALL" -C "$SRC_DIR" --strip-components=1 - rm "/tmp/$TARBALL" + tar xzf "$DOWNLOAD_DIR/$TARBALL" -C "$SRC_DIR" --strip-components=1 + printf '%s\n' "$expected_source_marker" > "$SOURCE_MARKER" + trap - EXIT + rm -rf "$DOWNLOAD_DIR" fi cd "$SRC_DIR" @@ -40,6 +65,7 @@ PATCH_SET=( "listen-success-exit.patch" "udp-listen-single-socket.patch" "disable-pktinfo.patch" + "disable-abortive-linger.patch" ) echo "==> Verifying Kandelo netcat portability patches..." for patch_name in "${PATCH_SET[@]}"; do @@ -76,6 +102,11 @@ if ! grep -q "/\\* # define USE_PKTINFO \\*/" src/netcat.h; then exit 1 fi +if ! grep -q "Kandelo cannot yet model abortive SO_LINGER" src/network.c; then + echo "ERROR: disable-abortive-linger.patch is missing from src/network.c" >&2 + exit 1 +fi + if [ ! -f Makefile ]; then echo "==> Configuring GNU Netcat for wasm32..." export ac_cv_func_malloc_0_nonnull=yes @@ -108,13 +139,32 @@ fi echo "==> Applying fork instrumentation metadata..." FORK_INSTRUMENT="$REPO_ROOT/scripts/run-wasm-fork-instrument.sh" -"$FORK_INSTRUMENT" "$NETCAT_BIN" -o "$NETCAT_BIN.instr" +(cd "$REPO_ROOT" && "$FORK_INSTRUMENT" "$NETCAT_BIN" -o "$NETCAT_BIN.instr") mv "$NETCAT_BIN.instr" "$NETCAT_BIN" mkdir -p "$BIN_DIR" cp "$NETCAT_BIN" "$BIN_DIR/nc.wasm" -source "$REPO_ROOT/scripts/install-local-binary.sh" -install_local_binary netcat "$BIN_DIR/nc.wasm" +if [ -n "${WASM_POSIX_DEP_OUT_DIR:-}" ]; then + # Resolver builds must publish only into the resolver-owned output + # directory. Apply the same artifact guards as install_local_binary without + # writing a local-binaries override into the source worktree. + # shellcheck source=/dev/null + source "$REPO_ROOT/scripts/wasm-artifact-guards.sh" + if ! wasm_is_binary "$BIN_DIR/nc.wasm"; then + echo "ERROR: refusing non-Wasm netcat artifact: $BIN_DIR/nc.wasm" >&2 + exit 1 + fi + wasm_require_no_legacy_asyncify "$BIN_DIR/nc.wasm" + wasm_require_fork_instrumentation_if_needed "$BIN_DIR/nc.wasm" + mkdir -p "$WASM_POSIX_DEP_OUT_DIR" + cp "$BIN_DIR/nc.wasm" "$WASM_POSIX_DEP_OUT_DIR/nc.wasm" + echo " installed $WASM_POSIX_DEP_OUT_DIR/nc.wasm (resolver scratch)" +else + # Direct developer builds retain the normal local resolver override. + # shellcheck source=/dev/null + source "$REPO_ROOT/scripts/install-local-binary.sh" + install_local_binary netcat "$BIN_DIR/nc.wasm" +fi ls -lh "$BIN_DIR/nc.wasm" diff --git a/packages/registry/netcat/build.toml b/packages/registry/netcat/build.toml index e19e4a7fa..e910c2d7c 100644 --- a/packages/registry/netcat/build.toml +++ b/packages/registry/netcat/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/netcat/build-netcat.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "2e6293a50ccf996b1a434aa701057b862a46c587" -revision = 1 +revision = 2 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/netcat/patches/disable-abortive-linger.patch b/packages/registry/netcat/patches/disable-abortive-linger.patch new file mode 100644 index 000000000..855af9bfe --- /dev/null +++ b/packages/registry/netcat/patches/disable-abortive-linger.patch @@ -0,0 +1,33 @@ +diff --git a/src/network.c b/src/network.c +index 23b2d2c..62e1f51 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -351,6 +351,8 @@ int netcat_socket_new(int domain, int type) + { + int sock, ret, sockopt; ++#if !defined(__wasm32__) + struct linger fix_ling; ++#endif + + sock = socket(domain, type, 0); + if (sock < 0) +@@ -359,5 +361,11 @@ int netcat_socket_new(int domain, int type) + + /* don't leave the socket in a TIME_WAIT state if we close the connection */ ++#if defined(__wasm32__) ++ /* Kandelo cannot yet model abortive SO_LINGER coherently across its ++ in-kernel, Node, and browser transports. The platform truthfully rejects ++ the enabled option, so retain ordinary FIN close semantics here. */ ++ ret = 0; ++#else + fix_ling.l_onoff = 1; + fix_ling.l_linger = 0; + ret = setsockopt(sock, SOL_SOCKET, SO_LINGER, &fix_ling, sizeof(fix_ling)); +@@ -367,6 +375,7 @@ int netcat_socket_new(int domain, int type) + close(sock); /* anyway the socket was created */ + return -2; + } ++#endif + + /* fix the socket options */ + sockopt = 1; diff --git a/packages/registry/node-vfs/build.toml b/packages/registry/node-vfs/build.toml index b7d7d9655..8df934459 100644 --- a/packages/registry/node-vfs/build.toml +++ b/packages/registry/node-vfs/build.toml @@ -21,7 +21,7 @@ inputs = [ ] repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 7 +revision = 8 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/perl-vfs/build.toml b/packages/registry/perl-vfs/build.toml index ea96416dc..539cae8c7 100644 --- a/packages/registry/perl-vfs/build.toml +++ b/packages/registry/perl-vfs/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/perl-vfs/build-perl-vfs.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/php/build-php.sh b/packages/registry/php/build-php.sh index 144b5ce14..1e30253ce 100755 --- a/packages/registry/php/build-php.sh +++ b/packages/registry/php/build-php.sh @@ -373,14 +373,6 @@ replace_once( "\t}\n" "}\n" "\n" - "static zend_always_inline void zend_wasm_arm_hard_timeout(zend_long seconds)\n" - "{\n" - "\tzend_hrtime_t now = zend_hrtime();\n" - "\tzend_hrtime_t hard_ns = zend_wasm_timeout_seconds_to_ns(seconds);\n" - "\tzend_wasm_timeout_deadline = 0;\n" - "\tzend_wasm_hard_timeout_deadline = zend_wasm_deadline_after(now, hard_ns);\n" - "}\n" - "\n" "static zend_always_inline bool zend_wasm_hard_timeout_expired(void)\n" "{\n" "\tzend_hrtime_t now;\n" @@ -488,11 +480,13 @@ replace_once( "\t}\n" "# endif\n" "\tzend_atomic_bool_store_ex(&EG(timed_out), false);\n" + "\t/* A responsive VM timeout follows POSIX zend_timeout(): disarm before shutdown. */\n" "\tzend_set_timeout_ex(0, 1);\n" "# if defined(__wasm32__) || defined(__wasm64__)\n" - "\tif (EG(hard_timeout) > 0) {\n" - "\t\tzend_wasm_arm_hard_timeout(EG(hard_timeout));\n" - "\t\t__wasm_posix_vm_interrupt_after(&EG(timed_out), &EG(vm_interrupt), EG(hard_timeout));\n" + "\t/* Give shutdown execution a fresh ordinary timeout phase. */\n" + "\tif (EG(timeout_seconds) > 0) {\n" + "\t\tzend_wasm_record_timeout_deadline(EG(timeout_seconds));\n" + "\t\t__wasm_posix_vm_interrupt_after(&EG(timed_out), &EG(vm_interrupt), EG(timeout_seconds));\n" "\t}\n" "# endif\n" "#endif\n\n" diff --git a/packages/registry/php/build.toml b/packages/registry/php/build.toml index 6fad09fa0..10e35d9d8 100644 --- a/packages/registry/php/build.toml +++ b/packages/registry/php/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/php/build-php.sh" repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 8 +revision = 10 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/php/test/browser/run-php.ts b/packages/registry/php/test/browser/run-php.ts index ac42a13db..b8dedbedd 100644 --- a/packages/registry/php/test/browser/run-php.ts +++ b/packages/registry/php/test/browser/run-php.ts @@ -43,7 +43,7 @@ async function runPhp( new SharedArrayBuffer(16 * 1024 * 1024, { maxByteLength: 64 * 1024 * 1024 }), 64 * 1024 * 1024, ); - for (const dir of ["/tmp", "/root", "/dev"]) ensureDir(memfs, dir); + for (const dir of ["/tmp", "/root", "/home", "/dev"]) ensureDir(memfs, dir); memfs.chmod("/tmp", 0o777); memfs.chmod("/root", 0o700); ensureDirRecursive(memfs, "/usr/local/bin"); diff --git a/packages/registry/python-vfs/build.toml b/packages/registry/python-vfs/build.toml index 41a8c3523..aa2d2379a 100644 --- a/packages/registry/python-vfs/build.toml +++ b/packages/registry/python-vfs/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/python-vfs/build-python-vfs.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/rootfs/build.toml b/packages/registry/rootfs/build.toml index 2d5643dbc..13ab3e23d 100644 --- a/packages/registry/rootfs/build.toml +++ b/packages/registry/rootfs/build.toml @@ -19,7 +19,7 @@ inputs = [ ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 5 +revision = 6 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/shell/build.toml b/packages/registry/shell/build.toml index 833d68a5f..f0261c162 100644 --- a/packages/registry/shell/build.toml +++ b/packages/registry/shell/build.toml @@ -24,7 +24,7 @@ inputs = [ ] repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 10 +revision = 11 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/wordpress/build.toml b/packages/registry/wordpress/build.toml index 4b8c21504..e37037b9c 100644 --- a/packages/registry/wordpress/build.toml +++ b/packages/registry/wordpress/build.toml @@ -6,6 +6,7 @@ inputs = [ "images/vfs/scripts/build-wp-vfs-image.sh", "images/vfs/scripts/build-wp-vfs-image.ts", "images/vfs/scripts/dinit-image-helpers.ts", + "images/rootfs/etc/services", "images/vfs/scripts/kandelo-demo-config.ts", "images/vfs/scripts/kandelo-demo-guides.ts", "images/vfs/scripts/opcache-prewarm.ts", @@ -28,7 +29,7 @@ inputs = [ ] repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 7 +revision = 8 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/wordpress/package.toml b/packages/registry/wordpress/package.toml index d872cb3a4..6281590bc 100644 --- a/packages/registry/wordpress/package.toml +++ b/packages/registry/wordpress/package.toml @@ -9,7 +9,7 @@ kernel_abi = 7 depends_on = [ "shell@0.1.0", "nginx@1.24.0", - "php@8.3.2", + "php@8.3.15", "dinit@0.19.4", "msmtpd@1.8.32", ] diff --git a/programs/ifhwaddr.c b/programs/ifhwaddr.c index e30bab372..00143f6f2 100644 --- a/programs/ifhwaddr.c +++ b/programs/ifhwaddr.c @@ -1,6 +1,8 @@ -/* Test program: retrieve network interface hardware (MAC) address via ioctl. - * Uses SIOCGIFCONF to enumerate interfaces, then SIOCGIFHWADDR on each. */ +/* Exercise Kandelo's virtual-interface ioctl and libc name/index contracts. */ +#include +#include #include +#include #include #include #include @@ -8,53 +10,278 @@ #include #include -int main(void) { - struct ifreq ifr[8]; - struct ifconf ifc; - int fd, i; +static int failures; - fd = socket(AF_INET, SOCK_DGRAM, 0); - if (fd < 0) { - perror("socket"); - return 1; +static void check(int condition, const char *description) +{ + if (!condition) { + printf("FAIL: %s (errno=%d)\n", description, errno); + failures++; } +} - ifc.ifc_req = ifr; - ifc.ifc_len = sizeof(ifr); +static void set_ifreq_name(struct ifreq *ifr, const char *name) +{ + memset(ifr, 0, sizeof(*ifr)); + strncpy(ifr->ifr_name, name, IF_NAMESIZE - 1); +} - if (ioctl(fd, SIOCGIFCONF, &ifc) < 0) { - perror("SIOCGIFCONF"); - close(fd); - return 1; +static unsigned char *ifreq_ipv4(struct ifreq *ifr) +{ + return (unsigned char *)&ifr->ifr_addr + 4; +} + +static int bytes_equal(const unsigned char *actual, + unsigned char a, unsigned char b, + unsigned char c, unsigned char d) +{ + return actual[0] == a && actual[1] == b && + actual[2] == c && actual[3] == d; +} + +static void check_ifconf(int fd) +{ + struct ifconf ifc = {0}; + struct ifreq interfaces[4]; + + check(ioctl(fd, SIOCGIFCONF, &ifc) == 0, + "SIOCGIFCONF size query succeeds"); + check(ifc.ifc_len == 2 * (int)sizeof(struct ifreq), + "SIOCGIFCONF size query reports two interfaces"); + printf("ifreq-size: %zu\n", sizeof(struct ifreq)); + + struct { + struct ifreq entry; + unsigned char guard[16]; + } bounded; + memset(&bounded, 0, sizeof(bounded)); + memset(bounded.guard, 0xa5, sizeof(bounded.guard)); + ifc.ifc_len = sizeof(bounded.entry); + ifc.ifc_req = &bounded.entry; + check(ioctl(fd, SIOCGIFCONF, &ifc) == 0, + "bounded SIOCGIFCONF succeeds"); + check(ifc.ifc_len == (int)sizeof(struct ifreq), + "bounded SIOCGIFCONF writes one complete entry"); + check(strcmp(bounded.entry.ifr_name, "lo") == 0, + "bounded SIOCGIFCONF returns lo first"); + for (size_t i = 0; i < sizeof(bounded.guard); i++) { + check(bounded.guard[i] == 0xa5, + "bounded SIOCGIFCONF preserves trailing guard bytes"); } - int n = ifc.ifc_len / sizeof(struct ifreq); - printf("interfaces: %d\n", n); + memset(interfaces, 0xcc, sizeof(interfaces)); + ifc.ifc_len = sizeof(interfaces); + ifc.ifc_req = interfaces; + check(ioctl(fd, SIOCGIFCONF, &ifc) == 0, + "full SIOCGIFCONF succeeds"); + int count = ifc.ifc_len / (int)sizeof(struct ifreq); + check(count == 2, "full SIOCGIFCONF returns two interfaces"); + check(strcmp(interfaces[0].ifr_name, "lo") == 0, + "SIOCGIFCONF names loopback lo"); + check(strcmp(interfaces[1].ifr_name, "eth0") == 0, + "SIOCGIFCONF names external interface eth0"); + check(interfaces[0].ifr_addr.sa_family == AF_INET, + "lo SIOCGIFCONF address uses AF_INET"); + check(interfaces[1].ifr_addr.sa_family == AF_INET, + "eth0 SIOCGIFCONF address uses AF_INET"); + check(bytes_equal(ifreq_ipv4(&interfaces[0]), 127, 0, 0, 1), + "lo SIOCGIFCONF address is loopback"); + unsigned char *eth = ifreq_ipv4(&interfaces[1]); + printf("ifconf: lo=127.0.0.1 eth0=%u.%u.%u.%u\n", + eth[0], eth[1], eth[2], eth[3]); - for (i = 0; i < n; i++) { - printf("name: %s\n", ifr[i].ifr_name); + ifc.ifc_len = sizeof(struct ifreq); + ifc.ifc_req = (struct ifreq *)(uintptr_t)-16; + errno = 0; + check(ioctl(fd, SIOCGIFCONF, &ifc) == -1 && errno == EFAULT, + "SIOCGIFCONF rejects an invalid nested buffer"); - if (ioctl(fd, SIOCGIFHWADDR, &ifr[i]) < 0) { - perror("SIOCGIFHWADDR"); - continue; - } + ifc.ifc_len = -1; + ifc.ifc_req = interfaces; + errno = 0; + check(ioctl(fd, SIOCGIFCONF, &ifc) == -1 && errno == EINVAL, + "SIOCGIFCONF rejects a negative length"); - unsigned char *mac = (unsigned char *)ifr[i].ifr_hwaddr.sa_data; - printf("mac: %02x:%02x:%02x:%02x:%02x:%02x\n", - mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + errno = 0; + check(ioctl(fd, SIOCGIFCONF, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFCONF rejects an invalid outer pointer"); +} - /* Check locally-administered bit */ - if (mac[0] & 0x02) { - printf("locally-administered: yes\n"); - } - /* Check non-zero */ - int all_zero = 1; - for (int j = 0; j < 6; j++) { - if (mac[j]) { all_zero = 0; break; } +static void check_name_index_ioctls(int fd) +{ + struct ifreq ifr; + + set_ifreq_name(&ifr, "lo"); + check(ioctl(fd, SIOCGIFINDEX, &ifr) == 0 && ifr.ifr_ifindex == 1, + "SIOCGIFINDEX maps lo to 1"); + set_ifreq_name(&ifr, "eth0"); + check(ioctl(fd, SIOCGIFINDEX, &ifr) == 0 && ifr.ifr_ifindex == 2, + "SIOCGIFINDEX maps eth0 to 2"); + set_ifreq_name(&ifr, "missing0"); + errno = 0; + check(ioctl(fd, SIOCGIFINDEX, &ifr) == -1 && errno == ENODEV, + "SIOCGIFINDEX rejects an unknown name"); + + memset(&ifr, 0, sizeof(ifr)); + ifr.ifr_ifindex = 1; + check(ioctl(fd, SIOCGIFNAME, &ifr) == 0 && + strcmp(ifr.ifr_name, "lo") == 0, + "SIOCGIFNAME maps 1 to lo"); + memset(&ifr, 0, sizeof(ifr)); + ifr.ifr_ifindex = 2; + check(ioctl(fd, SIOCGIFNAME, &ifr) == 0 && + strcmp(ifr.ifr_name, "eth0") == 0, + "SIOCGIFNAME maps 2 to eth0"); + memset(&ifr, 0, sizeof(ifr)); + ifr.ifr_ifindex = 99; + errno = 0; + check(ioctl(fd, SIOCGIFNAME, &ifr) == -1 && errno == ENODEV, + "SIOCGIFNAME rejects an unknown index"); + + errno = 0; + check(ioctl(fd, SIOCGIFINDEX, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFINDEX rejects an invalid pointer"); + errno = 0; + check(ioctl(fd, SIOCGIFNAME, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFNAME rejects an invalid pointer"); +} + +static void check_addresses(int fd) +{ + struct ifreq ifr; + + set_ifreq_name(&ifr, "lo"); + check(ioctl(fd, SIOCGIFADDR, &ifr) == 0, + "SIOCGIFADDR returns lo address"); + check(ifr.ifr_addr.sa_family == AF_INET && + bytes_equal(ifreq_ipv4(&ifr), 127, 0, 0, 1), + "SIOCGIFADDR reports 127.0.0.1 for lo"); + + set_ifreq_name(&ifr, "eth0"); + check(ioctl(fd, SIOCGIFADDR, &ifr) == 0, + "SIOCGIFADDR returns the backend eth0 address"); + check(ifr.ifr_addr.sa_family == AF_INET, + "eth0 SIOCGIFADDR address uses AF_INET"); + unsigned char *address = ifreq_ipv4(&ifr); + printf("eth0-address: %u.%u.%u.%u\n", + address[0], address[1], address[2], address[3]); + + set_ifreq_name(&ifr, "missing0"); + errno = 0; + check(ioctl(fd, SIOCGIFADDR, &ifr) == -1 && errno == ENODEV, + "SIOCGIFADDR rejects an unknown name"); + errno = 0; + check(ioctl(fd, SIOCGIFADDR, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFADDR rejects an invalid pointer"); +} + +static void check_hardware_addresses(int fd) +{ + struct ifreq ifr; + + set_ifreq_name(&ifr, "lo"); + check(ioctl(fd, SIOCGIFHWADDR, &ifr) == 0, + "SIOCGIFHWADDR returns lo hardware type"); + check(ifr.ifr_hwaddr.sa_family == ARPHRD_LOOPBACK, + "lo uses ARPHRD_LOOPBACK"); + int all_zero = 1; + for (int i = 0; i < 6; i++) { + if ((unsigned char)ifr.ifr_hwaddr.sa_data[i] != 0) all_zero = 0; + } + check(all_zero, "lo hardware address is all zero"); + + set_ifreq_name(&ifr, "eth0"); + check(ioctl(fd, SIOCGIFHWADDR, &ifr) == 0, + "SIOCGIFHWADDR returns eth0 hardware address"); + check(ifr.ifr_hwaddr.sa_family == ARPHRD_ETHER, + "eth0 uses ARPHRD_ETHER"); + unsigned char *mac = (unsigned char *)ifr.ifr_hwaddr.sa_data; + all_zero = 1; + for (int i = 0; i < 6; i++) if (mac[i]) all_zero = 0; + check(!all_zero, "eth0 hardware address is non-zero"); + check((mac[0] & 0x02) != 0 && (mac[0] & 0x01) == 0, + "eth0 hardware address is local unicast"); + printf("eth0-mac: %02x:%02x:%02x:%02x:%02x:%02x\n", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + + set_ifreq_name(&ifr, "missing0"); + errno = 0; + check(ioctl(fd, SIOCGIFHWADDR, &ifr) == -1 && errno == ENODEV, + "SIOCGIFHWADDR rejects an unknown name"); + errno = 0; + check(ioctl(fd, SIOCGIFHWADDR, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFHWADDR rejects an invalid pointer"); +} + +static void check_libc_name_index(void) +{ + errno = 0; + unsigned lo = if_nametoindex("lo"); + unsigned eth0 = if_nametoindex("eth0"); + unsigned missing = if_nametoindex("missing0"); + int missing_errno = errno; + check(lo == 1 && eth0 == 2, "if_nametoindex uses host ioctl mappings"); + check(missing == 0 && missing_errno == ENODEV, + "if_nametoindex rejects an unknown name"); + printf("libc-name-to-index: lo=%u eth0=%u missing=%u errno=%d\n", + lo, eth0, missing, missing_errno); + + char name[IF_NAMESIZE]; + check(if_indextoname(1, name) && strcmp(name, "lo") == 0, + "if_indextoname maps 1 to lo"); + check(if_indextoname(2, name) && strcmp(name, "eth0") == 0, + "if_indextoname maps 2 to eth0"); + errno = 0; + char *invalid = if_indextoname(99, name); + int invalid_errno = errno; + check(!invalid && invalid_errno == ENXIO, + "if_indextoname reports ENXIO for an unknown index"); + printf("libc-invalid-index: errno=%d\n", invalid_errno); + + struct if_nameindex *list = if_nameindex(); + check(list != NULL, "if_nameindex returns an interface list"); + if (list) { + int count = 0; + int saw_lo = 0; + int saw_eth0 = 0; + for (struct if_nameindex *entry = list; entry->if_index; entry++) { + printf("nameindex: %s=%u\n", entry->if_name, entry->if_index); + saw_lo |= entry->if_index == 1 && + strcmp(entry->if_name, "lo") == 0; + saw_eth0 |= entry->if_index == 2 && + strcmp(entry->if_name, "eth0") == 0; + count++; } - printf("non-zero: %s\n", all_zero ? "no" : "yes"); + check(count == 2 && saw_lo && saw_eth0, + "if_nameindex discovers lo and eth0 through ioctls"); + if_freenameindex(list); + } +} + +int main(void) +{ + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + perror("socket"); + return 1; } + check_ifconf(fd); + check_name_index_ioctls(fd); + check_addresses(fd); + check_hardware_addresses(fd); + check_libc_name_index(); + close(fd); + if (failures) { + printf("FAILURES: %d\n", failures); + return 1; + } + printf("PASS: virtual interface ioctl and libc contracts\n"); return 0; } diff --git a/scripts/build-programs.sh b/scripts/build-programs.sh index 7fe98b59d..36dfab67a 100755 --- a/scripts/build-programs.sh +++ b/scripts/build-programs.sh @@ -264,9 +264,12 @@ if [ -f "$SYSROOT64/lib/libc.a" ]; then -Wl,--export=__tls_align -Wl,--export=__stack_pointer -Wl,--export=__wasm_thread_init + -Wl,--export=__abi_version ) - for src in "$REPO_ROOT/programs/"hello64.c; do + for src in \ + "$REPO_ROOT/programs/"hello64.c \ + "$REPO_ROOT/programs/"ifhwaddr.c; do [ -f "$src" ] || continue local_name=$(basename "$src" .c) echo " Compiling $local_name (wasm64)..." diff --git a/scripts/run-php-upstream-node-chunks.sh b/scripts/run-php-upstream-node-chunks.sh index 19664df4c..efab5bc5f 100755 --- a/scripts/run-php-upstream-node-chunks.sh +++ b/scripts/run-php-upstream-node-chunks.sh @@ -269,7 +269,7 @@ while [ "$offset" -lt "$total" ]; do --timeout "$timeout_ms" \ --host-reset-interval "$host_reset_interval" \ --allow-unsupported \ - "${extra_args[@]}" \ + ${extra_args[@]+"${extra_args[@]}"} \ --json \ > "$jsonl" 2> "$stderr" status=$? diff --git a/scripts/run-php-upstream-tests.ts b/scripts/run-php-upstream-tests.ts index 0a5b8509f..ac93bbf35 100644 --- a/scripts/run-php-upstream-tests.ts +++ b/scripts/run-php-upstream-tests.ts @@ -1035,6 +1035,12 @@ class NodePhpRunner implements PhpRunner { { mountPoint: "/php-src", hostPath: this.sourceRoot, + // The source tree is the PHPT workspace. Present it as owned by the + // guest account selected for this run so non-root FPM/CLI coverage + // exercises normal POSIX permission checks instead of a root-owned + // host mount. + uid: this.runUid, + gid: this.runGid, }, { mountPoint: "/kandelo-bin",