From 31a6482a161410021a401343f4c3a4c7145737b3 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Fri, 5 Jun 2026 04:46:45 +0000 Subject: [PATCH 01/29] fix: unlink inline symlinks without following targets (cherry picked from commit d746770e0cbec79db741052bc012ab486d71000d) --- host/src/vfs/sharedfs-vendor.ts | 15 ++++++++++++++- host/test/symlink.test.ts | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index a8488ed888..2e6078ea92 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -1452,7 +1452,20 @@ export class SharedFS { this.inodeWriteLock(childIno); const linkCount = this.r32(childOff + INO_LINK_COUNT); if (linkCount <= 1) { - this.inodeTruncate(childIno, 0); + const size = this.r64(childOff + INO_SIZE); + if ((mode & S_IFMT) === S_IFLNK && size <= INLINE_SYMLINK_SIZE) { + // Short symlink targets live in the inode direct-pointer area. Unlink + // removes the symlink inode itself, not the inline target bytes as + // block pointers. + this.u8.fill( + 0, + childOff + INO_DIRECT, + childOff + INO_DIRECT + INLINE_SYMLINK_SIZE, + ); + this.w64(childOff + INO_SIZE, 0); + } else { + this.inodeTruncate(childIno, 0); + } this.w32(childOff + INO_LINK_COUNT, 0); this.inodeWriteUnlock(childIno); this.inodeFree(childIno); diff --git a/host/test/symlink.test.ts b/host/test/symlink.test.ts index f848dd6772..a5674d40af 100644 --- a/host/test/symlink.test.ts +++ b/host/test/symlink.test.ts @@ -51,6 +51,22 @@ describe("symlink and lstat", () => { expect(st.mode & S_IFMT).toBe(S_IFLNK); }); + it("unlink removes a dangling symlink itself", () => { + const mfs = createMemfs(); + + const fd = mfs.open("/target.txt", O_WRONLY | O_CREAT | O_TRUNC, 0o644); + mfs.close(fd); + mfs.symlink("target.txt", "/link.txt"); + + mfs.unlink("/target.txt"); + expect(mfs.lstat("/link.txt").mode & S_IFMT).toBe(S_IFLNK); + + // POSIX unlink(2) unlinks the directory entry named by path. When path is + // a symlink, it removes the link inode and does not follow the target. + mfs.unlink("/link.txt"); + expect(() => mfs.lstat("/link.txt")).toThrow(); + }); + it("readlink returns the symlink target", () => { const mfs = createMemfs(); mfs.symlink("/some/path", "/mylink"); From b91b08033090147d7493d1db8515f9e77d31f1db Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Mon, 15 Jun 2026 11:44:32 +0000 Subject: [PATCH 02/29] fix: preserve instrumented wasm file permissions (cherry picked from commit 6f9ccbc2d345a2855382d0ddaceb6567c1203d27) --- crates/fork-instrument/src/main.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/fork-instrument/src/main.rs b/crates/fork-instrument/src/main.rs index dde9b50a96..29023af5f4 100644 --- a/crates/fork-instrument/src/main.rs +++ b/crates/fork-instrument/src/main.rs @@ -13,7 +13,9 @@ use anyhow::{Context, Result}; use clap::Parser; use std::fs; -use std::path::PathBuf; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; use fork_instrument::{Options, analyze, instrument}; @@ -71,10 +73,31 @@ fn main() -> Result<()> { fs::write(output_path, &output) .with_context(|| format!("writing output: {}", output_path.display()))?; + preserve_input_permissions(&cli.input, output_path)?; Ok(()) } +#[cfg(unix)] +fn preserve_input_permissions(input_path: &Path, output_path: &Path) -> Result<()> { + let input_mode = fs::metadata(input_path) + .with_context(|| format!("stat input for permissions: {}", input_path.display()))? + .permissions() + .mode(); + let mut output_permissions = fs::metadata(output_path) + .with_context(|| format!("stat output for permissions: {}", output_path.display()))? + .permissions(); + output_permissions.set_mode(input_mode); + fs::set_permissions(output_path, output_permissions) + .with_context(|| format!("setting output permissions: {}", output_path.display()))?; + Ok(()) +} + +#[cfg(not(unix))] +fn preserve_input_permissions(_input_path: &Path, _output_path: &Path) -> Result<()> { + Ok(()) +} + fn print_analysis_json(analysis: &fork_instrument::Analysis) { // Hand-rolled JSON to avoid a serde dependency for a tiny output. // Format is one-entry-per-line array of `{name, is_import}` objects. From e380ecdf1417ddad8643ec3cf8bf3baf075161eb Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Mon, 15 Jun 2026 14:24:09 +0000 Subject: [PATCH 03/29] fix: preserve omitted host utimens timestamps (cherry picked from commit 4956be4e51e2626d190dbd1cf1e0f4e5b34b1a18) --- host/src/platform/native-metadata.ts | 24 +++++++++++++++++++++--- host/src/platform/node.ts | 23 ++++++++++++++++++++--- host/src/vfs/host-fs.ts | 26 +++++++++++++++++++++++--- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/host/src/platform/native-metadata.ts b/host/src/platform/native-metadata.ts index b90463a3a3..ecfa4779f8 100644 --- a/host/src/platform/native-metadata.ts +++ b/host/src/platform/native-metadata.ts @@ -11,6 +11,8 @@ interface VirtualMetadata { mode?: number; uid?: number; gid?: number; + atimeMs?: number; + mtimeMs?: number; ctimeMs?: number; } @@ -36,9 +38,11 @@ export class NativeMetadataOverlay { uid: metadata?.uid ?? 0, gid: metadata?.gid ?? 0, size: s.size, - atimeMs: s.atimeMs, - mtimeMs: s.mtimeMs, - ctimeMs: metadata?.ctimeMs ?? s.ctimeMs, + atimeMs: metadata?.atimeMs ?? s.atimeMs, + mtimeMs: metadata?.mtimeMs ?? s.mtimeMs, + ctimeMs: metadata?.ctimeMs === undefined + ? s.ctimeMs + : Math.max(metadata.ctimeMs, s.ctimeMs), }; } @@ -55,6 +59,20 @@ export class NativeMetadataOverlay { metadata.ctimeMs = Date.now(); } + utimens(s: Stats, atimeMs: number, mtimeMs: number, ctimeMs = Date.now()): void { + const metadata = this.metadataFor(s); + metadata.atimeMs = atimeMs; + metadata.mtimeMs = mtimeMs; + metadata.ctimeMs = Math.max(metadata.ctimeMs ?? 0, ctimeMs); + } + + noteNativeContentChange(s: Stats): void { + const metadata = this.entries.get(this.key(s)); + if (metadata === undefined) return; + delete metadata.atimeMs; + delete metadata.mtimeMs; + } + forget(s: Stats): void { this.entries.delete(this.key(s)); } diff --git a/host/src/platform/node.ts b/host/src/platform/node.ts index a5f82b3670..f82f0bdab7 100644 --- a/host/src/platform/node.ts +++ b/host/src/platform/node.ts @@ -97,6 +97,7 @@ export class NodePlatformIO implements PlatformIO { ): number { const pos = offset ?? this.fdPositions.get(handle) ?? 0; const bytesWritten = fs.writeSync(handle, buffer, 0, length, pos); + if (bytesWritten > 0) this.metadata.noteNativeContentChange(fs.fstatSync(handle)); if (offset === null) { this.fdPositions.set(handle, pos + bytesWritten); } @@ -218,9 +219,24 @@ export class NodePlatformIO implements PlatformIO { } utimensat(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { - const atime = atimeSec + atimeNsec / 1e9; - const mtime = mtimeSec + mtimeNsec / 1e9; - fs.utimesSync(this.rewritePath(path), atime, mtime); + const nativePath = this.rewritePath(path); + if (atimeNsec === UTIME_OMIT && mtimeNsec === UTIME_OMIT) return; + + const stat = fs.statSync(nativePath); + const current = this.metadata.toStatResult(stat); + const nowMs = Date.now(); + const atimeMs = atimeNsec === UTIME_OMIT + ? current.atimeMs + : atimeNsec === UTIME_NOW + ? nowMs + : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); + const mtimeMs = mtimeNsec === UTIME_OMIT + ? current.mtimeMs + : mtimeNsec === UTIME_NOW + ? nowMs + : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); + fs.utimesSync(nativePath, atimeMs / 1000, mtimeMs / 1000); + this.metadata.utimens(stat, atimeMs, mtimeMs, fs.statSync(nativePath).ctimeMs); } opendir(path: string): number { @@ -258,6 +274,7 @@ export class NodePlatformIO implements PlatformIO { ftruncate(handle: number, length: number): void { fs.ftruncateSync(handle, length); + this.metadata.noteNativeContentChange(fs.fstatSync(handle)); } fsync(handle: number): void { diff --git a/host/src/vfs/host-fs.ts b/host/src/vfs/host-fs.ts index 11d57d883f..e63250484e 100644 --- a/host/src/vfs/host-fs.ts +++ b/host/src/vfs/host-fs.ts @@ -12,6 +12,9 @@ import { NativeMetadataOverlay } from "../platform/native-metadata"; import type { FileSystemBackend, DirEntry } from "./types"; import { DEFAULT_STATFS_BLOCK_SIZE, DEFAULT_STATFS_NAMELEN } from "../statfs"; +const UTIME_NOW = 0x3fffffff; +const UTIME_OMIT = 0x3ffffffe; + /** * Translate Linux/POSIX open flags (as used by musl libc) to the * platform-native flag values that Node.js `fs.openSync` expects. @@ -160,6 +163,7 @@ export class HostFileSystem implements FileSystemBackend { ): number { const pos = offset ?? this.fdPositions.get(handle) ?? 0; const bytesWritten = fs.writeSync(handle, buffer, 0, length, pos); + if (bytesWritten > 0) this.metadata.noteNativeContentChange(fs.fstatSync(handle)); if (offset === null) { this.fdPositions.set(handle, pos + bytesWritten); } @@ -191,6 +195,7 @@ export class HostFileSystem implements FileSystemBackend { ftruncate(handle: number, length: number): void { fs.ftruncateSync(handle, length); + this.metadata.noteNativeContentChange(fs.fstatSync(handle)); } fsync(handle: number): void { @@ -274,9 +279,24 @@ export class HostFileSystem implements FileSystemBackend { } utimensat(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { - const atime = atimeSec + atimeNsec / 1e9; - const mtime = mtimeSec + mtimeNsec / 1e9; - fs.utimesSync(this.safePath(path), atime, mtime); + const nativePath = this.safePath(path); + if (atimeNsec === UTIME_OMIT && mtimeNsec === UTIME_OMIT) return; + + const stat = fs.statSync(nativePath); + const current = this.metadata.toStatResult(stat); + const nowMs = Date.now(); + const atimeMs = atimeNsec === UTIME_OMIT + ? current.atimeMs + : atimeNsec === UTIME_NOW + ? nowMs + : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); + const mtimeMs = mtimeNsec === UTIME_OMIT + ? current.mtimeMs + : mtimeNsec === UTIME_NOW + ? nowMs + : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); + fs.utimesSync(nativePath, atimeMs / 1000, mtimeMs / 1000); + this.metadata.utimens(stat, atimeMs, mtimeMs, fs.statSync(nativePath).ctimeMs); } // ── Directory iteration ───────────────────────────────────── From 0708ad4dc92ff4360f86ce44edb77445fc2df4d6 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Fri, 19 Jun 2026 03:03:20 -0400 Subject: [PATCH 04/29] fix: align host-backed mount path resolution --- host/src/vfs/host-fs.ts | 188 +++++++++++++++++++++++++++++++++++----- 1 file changed, 165 insertions(+), 23 deletions(-) diff --git a/host/src/vfs/host-fs.ts b/host/src/vfs/host-fs.ts index e63250484e..0359385b34 100644 --- a/host/src/vfs/host-fs.ts +++ b/host/src/vfs/host-fs.ts @@ -86,31 +86,170 @@ export function nativeStatfs(path: string): StatfsResult { export class HostFileSystem implements FileSystemBackend { private rootPath: string; + private guestMountPoint: string; private fdPositions = new Map(); private dirHandles = new Map(); private nextDirHandle = 1; private metadata = new NativeMetadataOverlay(); - - constructor(rootPath: string) { - this.rootPath = nodePath.resolve(rootPath); + private dirPathCache = new Map(); + private readonly maxDirPathCacheEntries = 4096; + + constructor(rootPath: string, guestMountPoint = "/") { + const resolvedRoot = nodePath.resolve(rootPath); + this.rootPath = fs.existsSync(resolvedRoot) + ? fs.realpathSync(resolvedRoot) + : resolvedRoot; + this.guestMountPoint = this.normalizeGuestMountPoint(guestMountPoint); } /** - * Resolve a mount-relative path to an absolute host path, - * ensuring it stays within `rootPath`. + * Resolve a mount-relative guest path to an absolute host path, ensuring it + * stays within `rootPath`. + * + * This intentionally resolves components one at a time instead of using + * `path.resolve()`. POSIX pathname resolution must look up an intermediate + * component before a following `..` can step back out of it: + * `existing/missing/../file` fails with ENOENT because `missing` is looked + * up as a directory first. Lexical normalization would incorrectly collapse + * that to `existing/file`. + * + * Native symlink targets are stored as guest strings. When following a + * symlink whose target is absolute and still inside this mount, translate it + * back to a mount-relative path before continuing. This preserves readlink(2) + * output while allowing stat/open/chmod to follow absolute in-guest links. */ - private safePath(relative: string): string { - const resolved = nodePath.resolve( - this.rootPath, - relative.replace(/^\//, ""), - ); - if ( - resolved !== this.rootPath && - !resolved.startsWith(this.rootPath + nodePath.sep) - ) { + private safePath(relative: string, followFinal = true): string { + const hadTrailingSlash = relative.length > 1 && /\/+$/.test(relative); + const originalParts = this.pathParts(relative); + let current = this.rootPath; + let pending = [...originalParts]; + let processed: string[] = []; + let symlinkDepth = 0; + let cacheable = !originalParts.includes(".."); + + if (cacheable) { + for (let i = originalParts.length; i > 0; i--) { + const cached = this.dirPathCache.get(originalParts.slice(0, i).join("/")); + if (cached === undefined) continue; + current = cached; + pending = originalParts.slice(i); + processed = originalParts.slice(0, i); + break; + } + } + + while (pending.length > 0) { + const part = pending.shift()!; + if (part === ".") continue; + if (part === "..") { + cacheable = false; + if (current === this.rootPath) { + throw new Error("EACCES: path traversal blocked"); + } + current = nodePath.dirname(current); + processed.pop(); + continue; + } + + const candidate = nodePath.join(current, part); + const isFinal = pending.length === 0; + const shouldFollow = !isFinal || followFinal; + + let lst: fs.Stats | null = null; + try { + lst = fs.lstatSync(candidate); + } catch (err: any) { + if (isFinal && err?.code === "ENOENT") { + current = candidate; + break; + } + throw err; + } + + if (shouldFollow && lst.isSymbolicLink()) { + cacheable = false; + if (++symlinkDepth > 40) throw new Error("ELOOP: too many symbolic links"); + const target = fs.readlinkSync(candidate, "utf8"); + if (target.startsWith("/")) { + const mountRelative = this.guestAbsoluteToMountRelative(target); + if (mountRelative === null) { + throw new Error("EACCES: absolute symlink target escapes mount"); + } + current = this.rootPath; + pending = [...this.pathParts(mountRelative), ...pending]; + } else { + pending = [...this.pathParts(target), ...pending]; + } + continue; + } + + if (!isFinal && !lst.isDirectory()) { + throw new Error("ENOTDIR: not a directory"); + } + + if (!isFinal) { + current = fs.realpathSync(candidate); + this.assertWithinRoot(current); + processed.push(part); + if (cacheable) this.setCachedDirPath(processed, current); + } else { + current = candidate; + } + } + + if (hadTrailingSlash && current !== this.rootPath && !current.endsWith(nodePath.sep)) { + // Keep a final separator for native fs calls. POSIX requires a + // trailing slash to resolve the preceding component as a directory; the + // native call then returns ENOTDIR for regular files while still + // permitting operations such as mkdir("new-dir/"). + current += nodePath.sep; + } + this.assertWithinRoot(current); + return current; + } + + private setCachedDirPath(parts: string[], nativePath: string): void { + if (parts.length === 0) return; + this.dirPathCache.set(parts.join("/"), nativePath); + if (this.dirPathCache.size > this.maxDirPathCacheEntries) { + const oldest = this.dirPathCache.keys().next().value; + if (oldest !== undefined) this.dirPathCache.delete(oldest); + } + } + + private clearDirPathCache(): void { + this.dirPathCache.clear(); + } + + private normalizeGuestMountPoint(mountPoint: string): string { + if (!mountPoint.startsWith("/")) mountPoint = `/${mountPoint}`; + return mountPoint !== "/" && mountPoint.endsWith("/") + ? mountPoint.slice(0, -1) + : mountPoint; + } + + private pathParts(path: string): string[] { + return path + .replace(/^\/+/, "") + .split("/") + .filter((part) => part.length > 0 && part !== "."); + } + + private guestAbsoluteToMountRelative(path: string): string | null { + if (this.guestMountPoint === "/") return path; + if (path === this.guestMountPoint) return "/"; + if (path.startsWith(`${this.guestMountPoint}/`)) { + return path.slice(this.guestMountPoint.length) || "/"; + } + return null; + } + + private assertWithinRoot(path: string): void { + const rel = nodePath.relative(this.rootPath, path); + if (rel === "") return; + if (rel.startsWith("..") || nodePath.isAbsolute(rel)) { throw new Error("EACCES: path traversal blocked"); } - return resolved; } private toStatResult(s: fs.Stats): StatResult { @@ -127,7 +266,7 @@ export class HostFileSystem implements FileSystemBackend { // ── File handle operations ─────────────────────────────────── open(path: string, flags: number, mode: number): number { - const nativePath = this.safePath(path); + const nativePath = this.safePath(path, (flags & 0o400000) === 0); const created = (flags & 0o100) !== 0 && !fs.existsSync(nativePath); const fd = fs.openSync(nativePath, translateOpenFlags(flags), mode); if (created) this.metadata.chmod(fs.fstatSync(fd), mode); @@ -217,7 +356,7 @@ export class HostFileSystem implements FileSystemBackend { } lstat(path: string): StatResult { - return this.toStatResult(fs.lstatSync(this.safePath(path))); + return this.toStatResult(fs.lstatSync(this.safePath(path, false))); } statfs(path: string): StatfsResult { @@ -231,26 +370,29 @@ export class HostFileSystem implements FileSystemBackend { } rmdir(path: string): void { - const nativePath = this.safePath(path); + const nativePath = this.safePath(path, false); const stat = fs.lstatSync(nativePath); fs.rmdirSync(nativePath); + this.clearDirPathCache(); this.metadata.forget(stat); } unlink(path: string): void { - const nativePath = this.safePath(path); + const nativePath = this.safePath(path, false); const stat = fs.lstatSync(nativePath); fs.unlinkSync(nativePath); + if (stat.isSymbolicLink()) this.clearDirPathCache(); if (stat.nlink <= 1) this.metadata.forget(stat); } rename(oldPath: string, newPath: string): void { - const nativeNewPath = this.safePath(newPath); + const nativeNewPath = this.safePath(newPath, false); let replaced: fs.Stats | undefined; try { replaced = fs.lstatSync(nativeNewPath); } catch {} - fs.renameSync(this.safePath(oldPath), nativeNewPath); + fs.renameSync(this.safePath(oldPath, false), nativeNewPath); + this.clearDirPathCache(); if (replaced !== undefined && replaced.nlink <= 1) this.metadata.forget(replaced); } @@ -259,11 +401,11 @@ export class HostFileSystem implements FileSystemBackend { } symlink(target: string, path: string): void { - fs.symlinkSync(target, this.safePath(path)); + fs.symlinkSync(target, this.safePath(path, false)); } readlink(path: string): string { - return fs.readlinkSync(this.safePath(path), "utf8"); + return fs.readlinkSync(this.safePath(path, false), "utf8"); } chmod(path: string, mode: number): void { From fc48786829536000ea10592dc48dba2940f10f9d Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Tue, 16 Jun 2026 05:11:58 +0000 Subject: [PATCH 05/29] fix: carry host mount and worker resource policy into Node Propagate mount ownership and native metadata through Node worker setup, preserve host-backed filesystem semantics, and apply the configured process-worker stack policy. --- host/src/node-kernel-host.ts | 10 +++++- host/src/node-kernel-protocol.ts | 8 ++++- host/src/node-kernel-worker-entry.ts | 13 +++++-- host/src/platform/native-metadata.ts | 9 +++-- host/src/vfs/host-fs.ts | 9 +++-- host/src/worker-adapter.ts | 36 ++++++++++++++++--- host/test/node-host-vfs-only-metadata.test.ts | 20 +++++++++++ 7 files changed, 92 insertions(+), 13 deletions(-) diff --git a/host/src/node-kernel-host.ts b/host/src/node-kernel-host.ts index a2fe31f688..2f2ac00220 100644 --- a/host/src/node-kernel-host.ts +++ b/host/src/node-kernel-host.ts @@ -88,7 +88,15 @@ export interface NodeKernelHostOptions { * to a VFS-only world yet. */ rootfsImage?: "default" | ArrayBuffer | Uint8Array; - extraMounts?: Array<{ mountPoint: string; hostPath: string; readonly?: boolean }>; + extraMounts?: Array<{ + mountPoint: string; + hostPath: string; + readonly?: boolean; + /** Virtual owner for existing host-backed mount entries. Defaults to root. */ + uid?: number; + /** Virtual group for existing host-backed mount entries. Defaults to root. */ + gid?: number; + }>; } export interface SpawnOptions { diff --git a/host/src/node-kernel-protocol.ts b/host/src/node-kernel-protocol.ts index 853fc19b49..3ce55aace5 100644 --- a/host/src/node-kernel-protocol.ts +++ b/host/src/node-kernel-protocol.ts @@ -37,7 +37,13 @@ export interface InitMessage { * (custom-io / legacy path). */ rootfsImage?: ArrayBuffer; - extraMounts?: Array<{ mountPoint: string; hostPath: string; readonly?: boolean }>; + extraMounts?: Array<{ + mountPoint: string; + hostPath: string; + readonly?: boolean; + uid?: number; + gid?: number; + }>; /** Attach a real-TCP backend (TcpNetworkBackend) to the worker's PlatformIO * so wasm programs can dial external hosts via Node `net.Socket`. */ enableTcpNetwork?: boolean; diff --git a/host/src/node-kernel-worker-entry.ts b/host/src/node-kernel-worker-entry.ts index 3ce9eeb1f4..f4d5dd56c3 100644 --- a/host/src/node-kernel-worker-entry.ts +++ b/host/src/node-kernel-worker-entry.ts @@ -553,7 +553,13 @@ function installDefaultCaBundle(fs: MemoryFileSystem): void { */ function buildVirtualPlatformIO( rootfsImage: ArrayBuffer, - extraMounts?: Array<{ mountPoint: string; hostPath: string; readonly?: boolean }>, + extraMounts?: Array<{ + mountPoint: string; + hostPath: string; + readonly?: boolean; + uid?: number; + gid?: number; + }>, ): VirtualPlatformIO { const bootSessionDir = mkdtempSync(join(tmpdir(), "wasm-posix-session-")); sessionDir = bootSessionDir; @@ -567,7 +573,10 @@ function buildVirtualPlatformIO( shmfs.chmod("/", 0o1777); const extras: MountConfig[] = (extraMounts ?? []).map((m) => ({ mountPoint: m.mountPoint, - backend: new HostFileSystem(m.hostPath), + backend: new HostFileSystem(m.hostPath, m.mountPoint, { + uid: m.uid, + gid: m.gid, + }), readonly: m.readonly, })); const mounts = [ diff --git a/host/src/platform/native-metadata.ts b/host/src/platform/native-metadata.ts index ecfa4779f8..bdc904a26b 100644 --- a/host/src/platform/native-metadata.ts +++ b/host/src/platform/native-metadata.ts @@ -26,6 +26,11 @@ interface VirtualMetadata { export class NativeMetadataOverlay { private readonly entries = new Map(); + constructor( + private readonly defaultUid = 0, + private readonly defaultGid = 0, + ) {} + toStatResult(s: Stats): StatResult { const metadata = this.entries.get(this.key(s)); return { @@ -35,8 +40,8 @@ export class NativeMetadataOverlay { ? s.mode : (s.mode & ~MODE_CHANGE_MASK) | (metadata.mode & MODE_CHANGE_MASK), nlink: s.nlink, - uid: metadata?.uid ?? 0, - gid: metadata?.gid ?? 0, + uid: metadata?.uid ?? this.defaultUid, + gid: metadata?.gid ?? this.defaultGid, size: s.size, atimeMs: metadata?.atimeMs ?? s.atimeMs, mtimeMs: metadata?.mtimeMs ?? s.mtimeMs, diff --git a/host/src/vfs/host-fs.ts b/host/src/vfs/host-fs.ts index 0359385b34..32427bbd2d 100644 --- a/host/src/vfs/host-fs.ts +++ b/host/src/vfs/host-fs.ts @@ -90,16 +90,21 @@ export class HostFileSystem implements FileSystemBackend { private fdPositions = new Map(); private dirHandles = new Map(); private nextDirHandle = 1; - private metadata = new NativeMetadataOverlay(); + private metadata: NativeMetadataOverlay; private dirPathCache = new Map(); private readonly maxDirPathCacheEntries = 4096; - constructor(rootPath: string, guestMountPoint = "/") { + constructor( + rootPath: string, + guestMountPoint = "/", + options: { uid?: number; gid?: number } = {}, + ) { const resolvedRoot = nodePath.resolve(rootPath); this.rootPath = fs.existsSync(resolvedRoot) ? fs.realpathSync(resolvedRoot) : resolvedRoot; this.guestMountPoint = this.normalizeGuestMountPoint(guestMountPoint); + this.metadata = new NativeMetadataOverlay(options.uid ?? 0, options.gid ?? 0); } /** diff --git a/host/src/worker-adapter.ts b/host/src/worker-adapter.ts index 19c919f482..635337f2b2 100644 --- a/host/src/worker-adapter.ts +++ b/host/src/worker-adapter.ts @@ -97,16 +97,43 @@ export class MockWorkerAdapter implements WorkerAdapter { // --- Node.js implementation --- -import { Worker } from "node:worker_threads"; +import { Worker, type WorkerOptions } from "node:worker_threads"; import { pathToFileURL } from "node:url"; import { createRequire } from "node:module"; import { existsSync } from "node:fs"; +// Wasm guest stacks consume the embedding worker's native stack when engines +// recurse through Wasm frames. Keep the default high enough for stack-heavy +// POSIX workloads while retaining an environment override for constrained +// embedders. +const DEFAULT_NODE_WORKER_STACK_SIZE_MB = 32; + function currentModuleUrl(): string { if (typeof __filename !== "undefined") return pathToFileURL(__filename).href; return import.meta.url; } +function nodeWorkerStackSizeMb(): number { + const raw = process.env.KANDELO_NODE_WORKER_STACK_SIZE_MB; + if (raw === undefined || raw === "") return DEFAULT_NODE_WORKER_STACK_SIZE_MB; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`invalid KANDELO_NODE_WORKER_STACK_SIZE_MB: ${raw}`); + } + return parsed; +} + +function nodeWorkerOptions(workerData: unknown, options: WorkerOptions = {}): WorkerOptions { + return { + ...options, + workerData, + resourceLimits: { + ...options.resourceLimits, + stackSizeMb: nodeWorkerStackSizeMb(), + }, + }; +} + export class NodeWorkerAdapter implements WorkerAdapter { private entryUrl: URL; private _compiledEntry: URL | false | undefined; @@ -153,7 +180,7 @@ export class NodeWorkerAdapter implements WorkerAdapter { // bootstrap which takes >500ms with 10+ concurrent workers). const compiledEntry = this.resolveCompiledEntry(); if (compiledEntry) { - const worker = new Worker(compiledEntry, { workerData }); + const worker = new Worker(compiledEntry, nodeWorkerOptions(workerData)); return new NodeWorkerHandle(worker); } @@ -169,10 +196,9 @@ export class NodeWorkerAdapter implements WorkerAdapter { `await import('${entryUrl}');`, ].join("\n"); - const worker = new Worker(bootstrap, { + const worker = new Worker(bootstrap, nodeWorkerOptions(workerData, { eval: true, - workerData, - }); + })); return new NodeWorkerHandle(worker); } } diff --git a/host/test/node-host-vfs-only-metadata.test.ts b/host/test/node-host-vfs-only-metadata.test.ts index c5b0ee8b3b..f0d3bc77e9 100644 --- a/host/test/node-host-vfs-only-metadata.test.ts +++ b/host/test/node-host-vfs-only-metadata.test.ts @@ -272,6 +272,26 @@ describe.each(backendFactories)("%s", (_name, makeCase) => { }); }); +describe("HostFileSystem default virtual ownership", () => { + it("can present existing host-backed files as owned by a chosen guest uid/gid", () => { + const root = makeTempRoot("wasm-posix-host-fs-default-owner-"); + const native = join(root, "owned-by-mount"); + writeFileSync(native, "data"); + const before = statSync(native); + + const backend = new HostFileSystem(root, "/", { uid: 65534, gid: 65533 }); + const virtual = backend.stat("/owned-by-mount"); + expect(virtual.uid).toBe(65534); + expect(virtual.gid).toBe(65533); + + backend.chown("/owned-by-mount", 1000, 1001); + const changed = backend.stat("/owned-by-mount"); + expect(changed.uid).toBe(1000); + expect(changed.gid).toBe(1001); + expectNativeMetadataUnchanged(native, before); + }); +}); + describe("VirtualPlatformIO on Node host mounts", () => { it("routes metadata operations to HostFileSystem as VFS-only changes", () => { const root = makeTempRoot("wasm-posix-virtual-platform-vfs-only-"); From 28149ddaf0d288e73f8b7d15f6006e8944eb25a9 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Tue, 16 Jun 2026 23:07:40 +0000 Subject: [PATCH 06/29] fix: reject invalid synthetic browser DNS names Validate hostnames before synthetic browser resolution, reserve the .invalid namespace for failures, and keep localhost aliases available without claiming general DNS reachability. --- host/src/networking/fetch-backend.ts | 7 +++-- host/src/networking/hostname.ts | 30 ++++++++++++++++++++++ host/src/networking/tls-network-backend.ts | 7 +++-- host/test/fetch-backend.test.ts | 28 +++++++++++++++++++- 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/host/src/networking/fetch-backend.ts b/host/src/networking/fetch-backend.ts index 83b126855a..8f26dbe49f 100644 --- a/host/src/networking/fetch-backend.ts +++ b/host/src/networking/fetch-backend.ts @@ -1,5 +1,8 @@ import type { NetworkIO } from "../types"; -import { parseNumericIpv4Hostname, validateDnsHostname } from "./hostname"; +import { + parseNumericIpv4Hostname, + validateSyntheticDnsHostname, +} from "./hostname"; /** Error with errno property for EAGAIN propagation to the kernel host imports. */ export class EagainError extends Error { @@ -230,7 +233,7 @@ export class FetchNetworkBackend implements NetworkIO { getaddrinfo(hostname: string): Uint8Array { const literalIp = parseNumericIpv4Hostname(hostname); if (literalIp) return literalIp; - validateDnsHostname(hostname); + validateSyntheticDnsHostname(hostname, this.options.hostAliases); // In the browser, return a synthetic IP. // The actual connection uses the Host header, not this IP. diff --git a/host/src/networking/hostname.ts b/host/src/networking/hostname.ts index a6de886544..9cd79ecdf1 100644 --- a/host/src/networking/hostname.ts +++ b/host/src/networking/hostname.ts @@ -72,3 +72,33 @@ export function validateDnsHostname(hostname: string): void { throw nameNotFoundError(hostname); } } + +/** + * Validate a name before a browser fetch/TLS backend assigns it a synthetic + * address. These backends defer the real lookup to fetch(), so they must reject + * names that the browser environment already knows cannot resolve. + */ +export function validateSyntheticDnsHostname( + hostname: string, + aliases?: Record, +): void { + validateDnsHostname(hostname); + + const absoluteName = hostname.endsWith(".") ? hostname.slice(0, -1) : hostname; + const lowerName = absoluteName.toLowerCase(); + if ( + aliases && + (Object.prototype.hasOwnProperty.call(aliases, absoluteName) || + Object.prototype.hasOwnProperty.call(aliases, lowerName)) + ) { + return; + } + if (lowerName === "localhost") return; + if ( + !lowerName.includes(".") || + lowerName === "invalid" || + lowerName.endsWith(".invalid") + ) { + throw nameNotFoundError(hostname); + } +} diff --git a/host/src/networking/tls-network-backend.ts b/host/src/networking/tls-network-backend.ts index aabf9dde9b..f53ed7abd6 100644 --- a/host/src/networking/tls-network-backend.ts +++ b/host/src/networking/tls-network-backend.ts @@ -15,7 +15,10 @@ import type { NetworkIO } from "../types"; import { EagainError } from "./fetch-backend"; -import { parseNumericIpv4Hostname, validateDnsHostname } from "./hostname"; +import { + parseNumericIpv4Hostname, + validateSyntheticDnsHostname, +} from "./hostname"; import { TLS_1_2_Connection } from "../../../packages/registry/openssl/src/tls/1_2/connection"; import { generateCertificate, @@ -232,7 +235,7 @@ export class TlsNetworkBackend implements NetworkIO { getaddrinfo(hostname: string): Uint8Array { const literalIp = parseNumericIpv4Hostname(hostname); if (literalIp) return literalIp; - validateDnsHostname(hostname); + validateSyntheticDnsHostname(hostname, this.dnsAliases); const ip = this.syntheticIp(hostname); const ipStr = this.ipKey(ip); diff --git a/host/test/fetch-backend.test.ts b/host/test/fetch-backend.test.ts index b13274a729..5eeb76aced 100644 --- a/host/test/fetch-backend.test.ts +++ b/host/test/fetch-backend.test.ts @@ -89,7 +89,7 @@ describe("FetchNetworkBackend", () => { }); describe("getaddrinfo", () => { - it("returns a 4-byte address for any hostname", () => { + it("returns a 4-byte address for DNS names that can be deferred to fetch", () => { const backend = new FetchNetworkBackend(); const addr = backend.getaddrinfo("example.com"); expect(addr.length).toBe(4); @@ -125,6 +125,19 @@ describe("FetchNetworkBackend", () => { expect(() => backend.getaddrinfo(".toto.toto.toto")).toThrow("ENOENT"); expect(() => backend.getaddrinfo(`www.${"x".repeat(100)}.com`)).toThrow("ENOENT"); }); + + it("rejects names the browser resolver cannot truthfully synthesize", () => { + const backend = new FetchNetworkBackend(); + expect(() => backend.getaddrinfo("dummy-host-name")).toThrow("ENOENT"); + expect(() => backend.getaddrinfo("totes.invalid")).toThrow("ENOENT"); + }); + + it("allows explicitly aliased unqualified names", () => { + const backend = new FetchNetworkBackend({ + hostAliases: { registry: "registry.npmjs.org" }, + }); + expect(backend.getaddrinfo("registry").length).toBe(4); + }); }); describe("connect", () => { @@ -216,6 +229,19 @@ describe("TlsNetworkBackend HTTP proxy path", () => { expect(() => backend.getaddrinfo(".toto.toto.toto")).toThrow("ENOENT"); expect(() => backend.getaddrinfo(`www.${"x".repeat(100)}.com`)).toThrow("ENOENT"); }); + + it("rejects special-use invalid and unqualified names", () => { + const backend = new TlsNetworkBackend(); + expect(() => backend.getaddrinfo("dummy-host-name")).toThrow("ENOENT"); + expect(() => backend.getaddrinfo("totes.invalid")).toThrow("ENOENT"); + }); + + it("allows explicitly aliased unqualified names", () => { + const backend = new TlsNetworkBackend({ + dnsAliases: { registry: "https://registry.npmjs.org" }, + }); + expect(backend.getaddrinfo("registry").length).toBe(4); + }); }); it("resets response state for keep-alive HTTP requests", async () => { From 83a32cbb36b497ae8b064dfa1270ed3beadc62bd Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Wed, 17 Jun 2026 02:48:20 +0000 Subject: [PATCH 07/29] fix: update sharedfs timestamps on content changes (cherry picked from commit 38a0e78feec47da771569d9478b50a711a4e729b) --- host/src/vfs/sharedfs-vendor.ts | 16 ++++++++++++++++ host/test/vfs.test.ts | 31 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index 2e6078ea92..59d4b30793 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -736,6 +736,11 @@ export class SharedFS { if (offset > size) { this.w64(inoOff + INO_SIZE, offset); } + if (totalWritten > 0) { + const now = Date.now(); + this.w64(inoOff + INO_MTIME, now); + this.w64(inoOff + INO_CTIME, now); + } return totalWritten; } @@ -808,13 +813,24 @@ export class SharedFS { private inodeTruncate(ino: number, newSize: number): void { const inoOff = this.inodeOffset(ino); const curSize = this.r64(inoOff + INO_SIZE); + const sizeChanged = newSize !== curSize; if (newSize >= curSize) { this.w64(inoOff + INO_SIZE, newSize); + if (sizeChanged) { + const now = Date.now(); + this.w64(inoOff + INO_MTIME, now); + this.w64(inoOff + INO_CTIME, now); + } return; } const keepBlocks = Math.ceil(newSize / BLOCK_SIZE); this.freeBlocksFrom(ino, keepBlocks); this.w64(inoOff + INO_SIZE, newSize); + if (sizeChanged) { + const now = Date.now(); + this.w64(inoOff + INO_MTIME, now); + this.w64(inoOff + INO_CTIME, now); + } } // ── Directory operations ───────────────────────────────────────── diff --git a/host/test/vfs.test.ts b/host/test/vfs.test.ts index c257b3d60a..a3a14140b5 100644 --- a/host/test/vfs.test.ts +++ b/host/test/vfs.test.ts @@ -523,6 +523,37 @@ describe("MemoryFileSystem", () => { mfs.close(fd); }); + it("updates mtime and ctime after file writes and truncates", () => { + const now = vi.spyOn(Date, "now"); + try { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + now.mockReturnValue(1_000); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_RDWR = 0x0002, + O_TRUNC = 0x0200; + const fd = mfs.open("/timestamps.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const initial = mfs.fstat(fd); + + now.mockReturnValue(5_000); + mfs.write(fd, new TextEncoder().encode("abc"), null, 3); + const afterWrite = mfs.fstat(fd); + expect(afterWrite.mtimeMs).toBe(5_000); + expect(afterWrite.ctimeMs).toBe(5_000); + expect(afterWrite.mtimeMs).toBeGreaterThan(initial.mtimeMs); + + now.mockReturnValue(9_000); + mfs.ftruncate(fd, 1); + const afterTruncate = mfs.fstat(fd); + expect(afterTruncate.mtimeMs).toBe(9_000); + expect(afterTruncate.ctimeMs).toBe(9_000); + expect(afterTruncate.mtimeMs).toBeGreaterThan(afterWrite.mtimeMs); + mfs.close(fd); + } finally { + now.mockRestore(); + } + }); + it("unlink removes a file", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); From 023111a5f513d6ac261ef7805b97d22bb8969b88 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Wed, 17 Jun 2026 07:24:08 +0000 Subject: [PATCH 08/29] fix: align browser network peeking and proxy fetches Preserve MSG_PEEK across browser networking backends and carry CORS proxy configuration through the browser worker protocol without consuming queued data. --- host/src/browser-kernel-host.ts | 5 ++ host/src/browser-kernel-protocol.ts | 4 + host/src/browser-kernel-worker-entry.ts | 1 + host/src/networking/fetch-backend.ts | 7 +- host/src/networking/tcp-backend.ts | 7 +- host/src/networking/tls-network-backend.ts | 99 ++++++++++++++++++++-- host/src/networking/virtual-network.ts | 7 +- host/test/fetch-backend.test.ts | 61 +++++++++++++ host/test/virtual-network.test.ts | 22 +++++ 9 files changed, 200 insertions(+), 13 deletions(-) diff --git a/host/src/browser-kernel-host.ts b/host/src/browser-kernel-host.ts index 58fe444ddc..534ad48796 100644 --- a/host/src/browser-kernel-host.ts +++ b/host/src/browser-kernel-host.ts @@ -85,6 +85,10 @@ export interface BrowserKernelOptions { syscallLogPtrWidth?: 4 | 8; /** Forwarded to TlsNetworkBackendOptions.dnsAliases. */ dnsAliases?: Record; + /** Forwarded to TlsNetworkBackendOptions.corsProxyUrl. Browser pages that + * are not controlled by Kandelo's service worker can use this to route + * guest outbound HTTP(S) through a same-origin proxy. */ + corsProxyUrl?: string; } /** Options for {@link BrowserKernel.boot}. */ @@ -393,6 +397,7 @@ export class BrowserKernel { enableSyscallLog: this.options.enableSyscallLog, syscallLogPtrWidth: this.options.syscallLogPtrWidth, dnsAliases: this.options.dnsAliases, + corsProxyUrl: this.options.corsProxyUrl, }, }; this.kernelWorkerHandle.postMessage(initMsg, [transferBuf]); diff --git a/host/src/browser-kernel-protocol.ts b/host/src/browser-kernel-protocol.ts index b64d202cb6..831aed52f7 100644 --- a/host/src/browser-kernel-protocol.ts +++ b/host/src/browser-kernel-protocol.ts @@ -59,6 +59,10 @@ export interface InitMessage { syscallLogPtrWidth?: 4 | 8; /** Forwarded to TlsNetworkBackendOptions.dnsAliases. */ dnsAliases?: Record; + /** Forwarded to TlsNetworkBackendOptions.corsProxyUrl for browser fetch + * backends that need a same-origin proxy to reach external HTTP(S) + * hosts. */ + corsProxyUrl?: string; }; } diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index 155f194893..8cb027b4db 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -698,6 +698,7 @@ async function handleInit(msg: Extract) { // production, keeping the browser networking path identical across modes. const tlsBackend = new TlsNetworkBackend({ dnsAliases: msg.config.dnsAliases, + corsProxyUrl: msg.config.corsProxyUrl, }); await tlsBackend.init(); io.network = tlsBackend; diff --git a/host/src/networking/fetch-backend.ts b/host/src/networking/fetch-backend.ts index 8f26dbe49f..9100e16b3e 100644 --- a/host/src/networking/fetch-backend.ts +++ b/host/src/networking/fetch-backend.ts @@ -14,6 +14,7 @@ const POLLIN = 0x0001; const POLLOUT = 0x0004; const POLLERR = 0x0008; const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; interface ConnectionState { hostname: string; @@ -175,7 +176,7 @@ export class FetchNetworkBackend implements NetworkIO { return data.length; } - recv(handle: number, maxLen: number, _flags: number): Uint8Array { + recv(handle: number, maxLen: number, flags: number): Uint8Array { const conn = this.connections.get(handle); if (!conn) throw new Error("ENOTCONN"); @@ -196,7 +197,9 @@ export class FetchNetworkBackend implements NetworkIO { if (len === 0) return new Uint8Array(0); const result = conn.responseBuf.slice(conn.responseOffset, conn.responseOffset + len); - conn.responseOffset += len; + if ((flags & MSG_PEEK) === 0) { + conn.responseOffset += len; + } return result; } diff --git a/host/src/networking/tcp-backend.ts b/host/src/networking/tcp-backend.ts index dfa996fb9a..cd69e2030b 100644 --- a/host/src/networking/tcp-backend.ts +++ b/host/src/networking/tcp-backend.ts @@ -8,6 +8,7 @@ const POLLIN = 0x0001; const POLLOUT = 0x0004; const POLLERR = 0x0008; const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; /** * Map a Node.js network error code to a POSIX errno value. @@ -138,7 +139,7 @@ export class TcpNetworkBackend implements NetworkIO { return data.length; } - recv(handle: number, maxLen: number, _flags: number): Uint8Array { + recv(handle: number, maxLen: number, flags: number): Uint8Array { const conn = this.connections.get(handle); if (!conn) throw new Error("ENOTCONN"); if (conn.error) throw conn.error; @@ -150,7 +151,9 @@ export class TcpNetworkBackend implements NetworkIO { conn.recvBuf.byteOffset, len, ); - conn.recvBuf = conn.recvBuf.subarray(len); + if ((flags & MSG_PEEK) === 0) { + conn.recvBuf = conn.recvBuf.subarray(len); + } return result; } diff --git a/host/src/networking/tls-network-backend.ts b/host/src/networking/tls-network-backend.ts index f53ed7abd6..18a0405720 100644 --- a/host/src/networking/tls-network-backend.ts +++ b/host/src/networking/tls-network-backend.ts @@ -26,6 +26,12 @@ import { type GeneratedCertificate, } from "../../../packages/registry/openssl/src/tls/certificates"; +const POLLIN = 0x0001; +const POLLOUT = 0x0004; +const POLLERR = 0x0008; +const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; + // ------------------------------------------------------------------ types interface HttpConnectionState { @@ -140,6 +146,17 @@ function formatHttpResponse( return result; } +function headersFromRawHeaderString(rawHeaders: string): Headers { + const headers = new Headers(); + for (const line of rawHeaders.split(/\r?\n/)) { + if (!line) continue; + const colon = line.indexOf(":"); + if (colon <= 0) continue; + headers.append(line.slice(0, colon).trim(), line.slice(colon + 1).trim()); + } + return headers; +} + function corsProxyFetchUrl(corsProxyUrl: string, targetUrl: string): string { if (targetUrl.startsWith(corsProxyUrl)) { return targetUrl; @@ -278,14 +295,14 @@ export class TlsNetworkBackend implements NetworkIO { return this.httpSend(conn, data); } - recv(handle: number, maxLen: number, _flags: number): Uint8Array { + recv(handle: number, maxLen: number, flags: number): Uint8Array { const conn = this.connections.get(handle); if (!conn) throw new Error("ENOTCONN"); if (conn.kind === "tls") { - return this.tlsRecv(conn, maxLen); + return this.tlsRecv(conn, maxLen, flags); } - return this.httpRecv(conn, maxLen); + return this.httpRecv(conn, maxLen, flags); } close(handle: number): void { @@ -425,14 +442,16 @@ export class TlsNetworkBackend implements NetworkIO { return data.length; } - private tlsRecv(conn: TlsConnectionState, maxLen: number): Uint8Array { + private tlsRecv(conn: TlsConnectionState, maxLen: number, flags: number): Uint8Array { if (conn.error) throw conn.error; // Check if we have encrypted data buffered from the TLS engine if (conn.clientDownstreamBuf.length > 0) { const n = Math.min(maxLen, conn.clientDownstreamBuf.length); const result = conn.clientDownstreamBuf.slice(0, n); - conn.clientDownstreamBuf = conn.clientDownstreamBuf.subarray(n); + if ((flags & MSG_PEEK) === 0) { + conn.clientDownstreamBuf = conn.clientDownstreamBuf.subarray(n); + } return result; } @@ -575,6 +594,32 @@ export class TlsNetworkBackend implements NetworkIO { const fetchBody: Uint8Array | undefined = body && body.length > 0 ? new Uint8Array(body) as Uint8Array : undefined; + const isWindowContext = typeof document !== "undefined"; + if (typeof XMLHttpRequest !== "undefined" && !isWindowContext) { + try { + const xhr = new XMLHttpRequest(); + xhr.open(method, url, false); + xhr.responseType = "arraybuffer"; + for (const [key, value] of fetchHeaders) { + xhr.setRequestHeader(key, value); + } + xhr.send(fetchBody); + conn.responseBuf = formatHttpResponse( + xhr.status, + xhr.statusText, + headersFromRawHeaderString(xhr.getAllResponseHeaders()), + xhr.response ?? new ArrayBuffer(0), + ); + conn.fetchDone = true; + conn.sendBuf = new Uint8Array(0); + return data.length; + } catch { + // Fall through to fetch when synchronous XHR is unavailable or rejected + // by the current browser context. Dedicated workers use the blocking + // path to match POSIX socket reads; other contexts retain async fetch. + } + } + const doFetch = async () => { try { const response = await fetch(url, { @@ -623,7 +668,7 @@ export class TlsNetworkBackend implements NetworkIO { return data.length; } - private httpRecv(conn: HttpConnectionState, maxLen: number): Uint8Array { + private httpRecv(conn: HttpConnectionState, maxLen: number, flags: number): Uint8Array { if (!conn.fetchDone) { throw new EagainError(); } @@ -639,10 +684,50 @@ export class TlsNetworkBackend implements NetworkIO { if (len === 0) return new Uint8Array(0); const result = conn.responseBuf.slice(conn.responseOffset, conn.responseOffset + len); - conn.responseOffset += len; + if ((flags & MSG_PEEK) === 0) { + conn.responseOffset += len; + } return result; } + poll(handle: number, events: number): number { + const conn = this.connections.get(handle); + if (!conn) throw Object.assign(new Error("ENOTCONN"), { errno: 107 }); + + let revents = 0; + if ((events & POLLOUT) !== 0 && (conn.kind === "http" || !conn.closed)) { + revents |= POLLOUT; + } + + if (conn.kind === "http") { + if (conn.fetchError) return revents | POLLERR; + if ( + (events & POLLIN) !== 0 && + conn.responseBuf && + conn.responseOffset < conn.responseBuf.length + ) { + revents |= POLLIN; + } + if ( + conn.fetchDone && + conn.responseBuf && + conn.responseOffset >= conn.responseBuf.length + ) { + revents |= POLLHUP; + } + return revents; + } + + if (conn.error) return revents | POLLERR; + if ((events & POLLIN) !== 0 && conn.clientDownstreamBuf.length > 0) { + revents |= POLLIN; + } + if (conn.closed && conn.clientDownstreamBuf.length === 0) { + revents |= POLLHUP; + } + return revents; + } + // ---- Utilities ---- private syntheticIp(hostname: string): Uint8Array { diff --git a/host/src/networking/virtual-network.ts b/host/src/networking/virtual-network.ts index 1e5b54cce4..e59f74294f 100644 --- a/host/src/networking/virtual-network.ts +++ b/host/src/networking/virtual-network.ts @@ -21,6 +21,7 @@ const POLLIN = 0x0001; const POLLOUT = 0x0004; const POLLERR = 0x0008; const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; const ANY = "0.0.0.0"; @@ -95,7 +96,7 @@ class VirtualTcpPeer implements TcpConnectionPeer { return data.length; } - recv(maxLen: number, _flags: number): Uint8Array { + recv(maxLen: number, flags: number): Uint8Array { if (this.reset) { const err = new Error("ECONNRESET") as Error & { errno?: number }; err.errno = ECONNRESET; @@ -104,7 +105,9 @@ class VirtualTcpPeer implements TcpConnectionPeer { if (this.recvBuf.length > 0) { const len = Math.min(maxLen, this.recvBuf.length); const out = this.recvBuf.slice(0, len); - this.recvBuf = this.recvBuf.slice(len); + if ((flags & MSG_PEEK) === 0) { + this.recvBuf = this.recvBuf.slice(len); + } return out; } if (!this.peer || this.peer.writeClosed) { diff --git a/host/test/fetch-backend.test.ts b/host/test/fetch-backend.test.ts index 5eeb76aced..5480c27dd7 100644 --- a/host/test/fetch-backend.test.ts +++ b/host/test/fetch-backend.test.ts @@ -4,6 +4,7 @@ import { TlsNetworkBackend, type TlsMitmConnection } from "../src/networking/tls const encoder = new TextEncoder(); const decoder = new TextDecoder(); +const MSG_PEEK = 0x0002; afterEach(() => { vi.restoreAllMocks(); @@ -83,6 +84,18 @@ class LoopbackMitmTls implements TlsMitmConnection { async close(): Promise {} } +async function waitForReadable( + backend: Pick, + handle: number, +): Promise { + const deadline = Date.now() + 1_000; + while (Date.now() < deadline) { + if ((backend.poll(handle, 0x0001) & 0x0001) !== 0) return; + await new Promise((resolve) => setTimeout(resolve, 0)); + } + throw new Error("timed out waiting for readable poll"); +} + describe("FetchNetworkBackend", () => { afterEach(() => { vi.restoreAllMocks(); @@ -181,6 +194,32 @@ describe("FetchNetworkBackend", () => { }); }); + it("honors MSG_PEEK without consuming buffered response bytes", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("hello"))); + const backend = new FetchNetworkBackend(); + const addr = backend.getaddrinfo("example.com"); + backend.connect(1, addr, 80); + backend.send( + 1, + encoder.encode("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"), + 0, + ); + + const first = decoder.decode(await recvWhenReady(backend, 1)); + expect(first).toContain("hello"); + + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("world"))); + backend.send( + 1, + encoder.encode("GET /2 HTTP/1.1\r\nHost: example.com\r\n\r\n"), + 0, + ); + await waitForReadable(backend, 1); + const peeked = decoder.decode(backend.recv(1, 4, MSG_PEEK)); + const consumed = decoder.decode(backend.recv(1, 4, 0)); + expect(peeked).toBe(consumed); + }); + describe("hostAliases", () => { it("rewrites the fetch target while preserving the request port", () => { const fetchMock = vi @@ -293,6 +332,28 @@ describe("TlsNetworkBackend HTTP proxy path", () => { expect(response.toLowerCase()).not.toContain("content-encoding"); expect(response.toLowerCase()).not.toContain("connection: close"); }); + + it("honors MSG_PEEK without consuming HTTP response bytes", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("peek-body"))); + const backend = new TlsNetworkBackend(); + const addr = backend.getaddrinfo("proxy.local"); + backend.connect(1, addr, 80); + + sendGet(backend, 1, "/peek"); + await recvWhenReady(backend, 1); + + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("second-body"))); + sendGet(backend, 1, "/peek2"); + await recvWhenReady(backend, 1); + + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("third-body"))); + sendGet(backend, 1, "/peek3"); + const peeked = decoder.decode((await recvWhenReady({ + recv: (handle, maxLen) => backend.recv(handle, maxLen, MSG_PEEK), + }, 1)).subarray(0, 8)); + const consumed = decoder.decode(backend.recv(1, 8, 0)); + expect(peeked).toBe(consumed); + }); }); describe("TlsNetworkBackend TLS MITM path", () => { diff --git a/host/test/virtual-network.test.ts b/host/test/virtual-network.test.ts index dd0fae50bb..44718adf8b 100644 --- a/host/test/virtual-network.test.ts +++ b/host/test/virtual-network.test.ts @@ -10,6 +10,7 @@ const POLLIN = 0x0001; const POLLOUT = 0x0004; const POLLERR = 0x0008; const POLLHUP = 0x0010; +const MSG_PEEK = 0x0002; describe("LocalVirtualNetwork", () => { it("resolves bounded legacy numeric IPv4 forms and valid DNS aliases", () => { @@ -56,6 +57,27 @@ describe("LocalVirtualNetwork", () => { expect(new TextDecoder().decode(client.recv(7, 16, 0))).toBe("pong"); }); + it("honors MSG_PEEK without consuming TCP stream data", () => { + const net = new LocalVirtualNetwork(); + const server = net.attachMachine({ id: "server", address: [10, 88, 0, 2] }); + const client = net.attachMachine({ id: "client", address: [10, 88, 0, 3] }); + let accepted: TcpConnectionPeer | null = null; + + expect(server.listenTcp!("srv:1", new Uint8Array([10, 88, 0, 2]), 8080, { + accept(peer) { + accepted = peer; + return 0; + }, + })).toBe(0); + + client.connect(7, new Uint8Array([10, 88, 0, 2]), 8080); + expect(accepted).not.toBeNull(); + accepted!.send(new TextEncoder().encode("peek-data"), 0); + + expect(new TextDecoder().decode(client.recv(7, 4, MSG_PEEK))).toBe("peek"); + expect(new TextDecoder().decode(client.recv(7, 9, 0))).toBe("peek-data"); + }); + it("reports refused TCP connects when no listener is bound", () => { const net = new LocalVirtualNetwork(); const client = net.attachMachine({ id: "client", address: [10, 88, 0, 3] }); From a996514819ede471d56d638d4f16795da99d1701 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Wed, 17 Jun 2026 09:11:49 +0000 Subject: [PATCH 09/29] fix: precompile exec and spawn modules before launch --- host/src/browser-kernel-worker-entry.ts | 27 +++++++++++++++++-------- host/src/node-kernel-worker-entry.ts | 26 ++++++++++++++++-------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index 8cb027b4db..88f71b3bcb 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -1150,11 +1150,12 @@ async function handleExec( if (!resolved) return -2; // ENOENT if ("errno" in resolved) return -resolved.errno; const { programBytes: bytes, argv: launchArgv } = resolved; - + let programModule: WebAssembly.Module; try { - await WebAssembly.compile(bytes); - } catch { - return -8; // ENOEXEC: reject malformed modules before changing old state + programModule = await WebAssembly.compile(bytes); + } catch (e) { + if (e instanceof WebAssembly.CompileError) return -8; // ENOEXEC + throw e; } const declaredAbi = extractAbiVersion(bytes); if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { @@ -1233,6 +1234,7 @@ async function handleExec( pid, ppid: 0, programBytes: bytes, + programModule, memory: newMemory, channelOffset: newChannelOffset, argv: launchArgv, @@ -1260,6 +1262,7 @@ async function handleExec( processes.set(pid, { memory: newMemory, programBytes: bytes, + programModule, worker: replacementWorker, argv: launchArgv, channelOffset: newChannelOffset, @@ -1353,11 +1356,17 @@ async function handlePosixSpawn( ): Promise { await waitForProcessTeardowns(); - // The child can receive a group-directed signal while the browser waits for - // an unrelated teardown. Keep the successful spawn and kernel-owned zombie, - // but never install a Worker for a process that has already terminated. - if (!kernelWorker.shouldLaunchPendingChild(childPid)) return 0; + let programModule: WebAssembly.Module; + try { + programModule = await WebAssembly.compile(programBytes); + } catch (e) { + if (e instanceof WebAssembly.CompileError) return -8; // ENOEXEC + throw e; + } + // Compilation and unrelated teardown waits yield to the event loop. Keep a + // successfully created zombie, but never resurrect it with a new Worker. + if (!kernelWorker.shouldLaunchPendingChild(childPid)) return 0; post({ type: "proc_event", kind: "spawn", pid: childPid }); const ptrWidth = detectPtrWidth(programBytes); @@ -1386,6 +1395,7 @@ async function handlePosixSpawn( pid: childPid, ppid: 0, programBytes, + programModule, memory: newMemory, channelOffset: newChannelOffset, argv, @@ -1399,6 +1409,7 @@ async function handlePosixSpawn( processes.set(childPid, { memory: newMemory, programBytes, + programModule, worker: newWorker, argv, channelOffset: newChannelOffset, diff --git a/host/src/node-kernel-worker-entry.ts b/host/src/node-kernel-worker-entry.ts index f4d5dd56c3..e86ea6a757 100644 --- a/host/src/node-kernel-worker-entry.ts +++ b/host/src/node-kernel-worker-entry.ts @@ -895,11 +895,12 @@ async function handleExec( if (!resolved) return -2; // ENOENT if ("errno" in resolved) return -resolved.errno; const { programBytes, argv: launchArgv } = resolved; - + let programModule: WebAssembly.Module; try { - await WebAssembly.compile(programBytes); - } catch { - return -8; // ENOEXEC: reject malformed modules before changing old state + programModule = await getCompiledProgramModule(programBytes); + } catch (e) { + if (e instanceof WebAssembly.CompileError) return -8; // ENOEXEC + throw e; } const declaredAbi = extractAbiVersion(programBytes); if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { @@ -965,6 +966,7 @@ async function handleExec( pid, ppid: 0, programBytes, + programModule, memory: newMemory, channelOffset: newChannelOffset, argv: launchArgv, @@ -993,6 +995,7 @@ async function handleExec( processes.set(pid, { memory: newMemory, programBytes, + programModule, worker: replacementWorker, channelOffset: newChannelOffset, ptrWidth: newPtrWidth, @@ -1100,12 +1103,17 @@ async function handlePosixSpawn( argv: string[], envp: string[], ): Promise { - // The kernel child is authoritative as soon as kernel_spawn_process returns. - // A group-directed signal may already have terminated it before the host - // installs a Worker. Preserve the successful spawn and its waitable zombie, + let programModule: WebAssembly.Module; + try { + programModule = await getCompiledProgramModule(programBytes); + } catch (e) { + if (e instanceof WebAssembly.CompileError) return -8; // ENOEXEC + throw e; + } + + // Compilation yields. Preserve a child that became a zombie while waiting, // but do not resurrect it by registering a new execution generation. if (!kernelWorker.shouldLaunchPendingChild(childPid)) return 0; - post({ type: "proc_event", kind: "spawn", pid: childPid }); const ptrWidth = detectPtrWidth(programBytes); @@ -1131,6 +1139,7 @@ async function handlePosixSpawn( pid: childPid, ppid: 0, programBytes, + programModule, memory, channelOffset, argv, @@ -1143,6 +1152,7 @@ async function handlePosixSpawn( processes.set(childPid, { memory, programBytes, + programModule, worker: newWorker, channelOffset, ptrWidth, From 0a4963101299cf833b61dd24a77e298f5c30d93a Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Wed, 17 Jun 2026 11:01:06 +0000 Subject: [PATCH 10/29] fix: honor exclusive creates in SharedFS Implement POSIX O_CREAT|O_EXCL handling in SharedFS so open fails with EEXIST when the final pathname already exists, including a final symlink. --- host/src/vfs/sharedfs-vendor.ts | 9 +++++++++ host/test/vfs.test.ts | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index 59d4b30793..60b0eae721 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -47,6 +47,7 @@ export const O_RDONLY = 0x0000; export const O_WRONLY = 0x0001; export const O_RDWR = 0x0002; export const O_CREAT = 0x0040; +export const O_EXCL = 0x0080; export const O_TRUNC = 0x0200; export const O_APPEND = 0x0400; export const O_DIRECTORY = 0x010000; @@ -1248,6 +1249,13 @@ export class SharedFS { open(path: string, flags: number, createMode: number = 0o644): number { const accMode = flags & O_ACCMODE; const creating = (flags & O_CREAT) !== 0; + const exclusive = (flags & O_EXCL) !== 0; + + if (creating && exclusive) { + const existing = this.pathResolve(path, false); + if (existing >= 0) throw new SFSError(EEXIST); + if (existing !== ENOENT) throw new SFSError(existing); + } let ino = this.pathResolve(path, true); @@ -1260,6 +1268,7 @@ export class SharedFS { const nameBytes = encoder.encode(name); const existing = this.dirLookup(parentIno, nameBytes); if (existing >= 0) { + if (exclusive) throw new SFSError(EEXIST); ino = existing; } else { const newIno = this.inodeAlloc(); diff --git a/host/test/vfs.test.ts b/host/test/vfs.test.ts index a3a14140b5..287204a509 100644 --- a/host/test/vfs.test.ts +++ b/host/test/vfs.test.ts @@ -509,6 +509,28 @@ describe("MemoryFileSystem", () => { expect(entries).toContain("file.txt"); }); + it("honors O_CREAT|O_EXCL by failing when the final path already exists", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_WRONLY = 0x0001, + O_CREAT = 0x0040, + O_EXCL = 0x0080; + + const fd = mfs.open("/exclusive.txt", O_WRONLY | O_CREAT | O_EXCL, 0o600); + mfs.close(fd); + + expect(() => + mfs.open("/exclusive.txt", O_WRONLY | O_CREAT | O_EXCL, 0o600), + ).toThrow(/File exists/); + + // POSIX open(O_CREAT|O_EXCL) must fail with EEXIST when the final path is + // a symbolic link, even if the symlink points at an existing regular file. + mfs.symlink("/exclusive.txt", "/exclusive-link.txt"); + expect(() => + mfs.open("/exclusive-link.txt", O_WRONLY | O_CREAT | O_EXCL, 0o600), + ).toThrow(/File exists/); + }); + it("stat returns correct size after writing", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); From 6ea06d0cf475522626a2e5ceaf25b16f17e039f6 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Wed, 17 Jun 2026 14:52:54 +0000 Subject: [PATCH 11/29] fix: keep virtual interface name APIs consistent (cherry picked from commit 20fae8b49d76d58e02af707963e7719654451e02) --- host/src/kernel-worker.ts | 74 +++++++++++++++++-- host/test/ifhwaddr.test.ts | 2 + .../src/network/wasm32posix/if_indextoname.c | 2 +- .../src/network/wasm32posix/if_nameindex.c | 4 +- .../src/network/wasm32posix/if_nametoindex.c | 2 +- programs/ifhwaddr.c | 6 ++ 6 files changed, 81 insertions(+), 9 deletions(-) diff --git a/host/src/kernel-worker.ts b/host/src/kernel-worker.ts index 6144bd2139..4d9f0a3887 100644 --- a/host/src/kernel-worker.ts +++ b/host/src/kernel-worker.ts @@ -204,9 +204,14 @@ const SIGCHLD = 17; const SIGALRM = 14; /** Network ioctl request codes */ +const SIOCGIFNAME = 0x8910; const SIOCGIFCONF = 0x8912; const SIOCGIFHWADDR = 0x8927; const SIOCGIFADDR = 0x8915; +const SIOCGIFINDEX = 0x8933; +const VIRTUAL_IFACE_NAME = "eth0"; +const VIRTUAL_IFACE_INDEX = 1; +const ENODEV = 19; /** Ioctl syscall number */ const SYS_IOCTL = ABI_SYSCALLS.Ioctl; @@ -3051,7 +3056,7 @@ export class CentralizedKernelWorker { return; } - // --- ioctl: intercept network interface ioctls (SIOCGIFCONF, SIOCGIFHWADDR) --- + // --- ioctl: intercept network interface ioctls --- // These require host-side handling because: // SIOCGIFCONF: struct ifconf contains a pointer to a process-memory buffer // SIOCGIFHWADDR: returns the virtual MAC address for this kernel instance @@ -3061,6 +3066,10 @@ export class CentralizedKernelWorker { this.handleIoctlIfconf(channel, origArgs); return; } + if (request === SIOCGIFNAME) { + this.handleIoctlIfname(channel, origArgs); + return; + } if (request === SIOCGIFHWADDR) { this.handleIoctlIfhwaddr(channel, origArgs); return; @@ -3069,6 +3078,10 @@ export class CentralizedKernelWorker { this.handleIoctlIfaddr(channel, origArgs); return; } + if (request === SIOCGIFINDEX) { + this.handleIoctlIfindex(channel, origArgs); + return; + } } // --- fcntl with struct flock pointer --- @@ -6057,8 +6070,8 @@ export class CentralizedKernelWorker { } // ---- Network interface ioctl host-side handlers ---- - // The kernel has a single virtual network interface ("eth0") with a random - // MAC address generated per kernel instance. + // The kernel has a single virtual network interface ("eth0") at ifindex 1 + // with a random MAC address generated per kernel instance. /** * Handle SIOCGIFCONF: enumerate network interfaces. @@ -6088,8 +6101,8 @@ export class CentralizedKernelWorker { const SIZEOF_IFREQ = 32; if (ifcLen >= SIZEOF_IFREQ && ifcBuf !== 0) { - // Write one ifreq entry for "eth0" into process memory at ifc_buf - const nameBytes = new TextEncoder().encode("eth0"); + // Write one ifreq entry for the virtual interface into process memory at ifc_buf + const nameBytes = new TextEncoder().encode(VIRTUAL_IFACE_NAME); processMem.set(nameBytes, ifcBuf); processMem.fill(0, ifcBuf + nameBytes.length, ifcBuf + 16); // pad ifr_name @@ -6112,6 +6125,30 @@ export class CentralizedKernelWorker { this.relistenChannel(channel); } + /** + * Handle SIOCGIFNAME: map an interface index to its name. + * struct ifreq at arg[2]: ifr_name[16] + union; ifr_ifindex lives at +16. + */ + private handleIoctlIfname(channel: ChannelInfo, origArgs: number[]): void { + const processView = new DataView(channel.memory.buffer); + const processMem = new Uint8Array(channel.memory.buffer); + const ifreqPtr = origArgs[2]; + const ifindex = processView.getInt32(ifreqPtr + 16, true); + + if (ifindex !== VIRTUAL_IFACE_INDEX) { + this.completeChannelRaw(channel, -ENODEV, ENODEV); + this.relistenChannel(channel); + return; + } + + const nameBytes = new TextEncoder().encode(VIRTUAL_IFACE_NAME); + processMem.set(nameBytes, ifreqPtr); + processMem.fill(0, ifreqPtr + nameBytes.length, ifreqPtr + 16); + + this.completeChannelRaw(channel, 0, 0); + this.relistenChannel(channel); + } + /** * Handle SIOCGIFHWADDR: get hardware (MAC) address for an interface. * struct ifreq at arg[2]: ifr_name[16] + ifr_hwaddr (struct sockaddr, 16 bytes) @@ -6155,6 +6192,33 @@ export class CentralizedKernelWorker { this.relistenChannel(channel); } + /** + * Handle SIOCGIFINDEX: map an interface name to its index. + * struct ifreq at arg[2]: ifr_name[16] + union; ifr_ifindex lives at +16. + */ + private handleIoctlIfindex(channel: ChannelInfo, origArgs: number[]): void { + const processView = new DataView(channel.memory.buffer); + const processMem = new Uint8Array(channel.memory.buffer); + const ifreqPtr = origArgs[2]; + const nul = processMem.indexOf(0, ifreqPtr); + const end = nul >= ifreqPtr && nul < ifreqPtr + 16 ? nul : ifreqPtr + 16; + // Browser TextDecoder rejects SharedArrayBuffer-backed views. Copy the + // guest ifr_name bytes before decoding; ioctl handlers must work for + // process memories backed by SAB. + const name = new TextDecoder().decode(new Uint8Array(processMem.subarray(ifreqPtr, end))); + + if (name !== VIRTUAL_IFACE_NAME) { + this.completeChannelRaw(channel, -ENODEV, ENODEV); + this.relistenChannel(channel); + return; + } + + processView.setInt32(ifreqPtr + 16, VIRTUAL_IFACE_INDEX, true); + + this.completeChannelRaw(channel, 0, 0); + this.relistenChannel(channel); + } + private handleWritev(channel: ChannelInfo, syscallNr: number, origArgs: number[]): void { const fd = origArgs[0]; const iovPtr = origArgs[1]; diff --git a/host/test/ifhwaddr.test.ts b/host/test/ifhwaddr.test.ts index 08c4eae00d..c5ff39a534 100644 --- a/host/test/ifhwaddr.test.ts +++ b/host/test/ifhwaddr.test.ts @@ -14,6 +14,8 @@ describe("SIOCGIFCONF / SIOCGIFHWADDR", () => { // Should find one interface named "eth0" expect(result.stdout).toContain("interfaces: 1"); expect(result.stdout).toContain("name: eth0"); + expect(result.stdout).toContain("index: 1"); + expect(result.stdout).toContain("index-name: eth0"); // MAC should be locally-administered and non-zero expect(result.stdout).toContain("locally-administered: yes"); diff --git a/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c b/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c index f933623bc8..2e2d3a462e 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c @@ -5,7 +5,7 @@ char *if_indextoname(unsigned index, char *name) { if (index == 1) - return strncpy(name, "lo", IF_NAMESIZE); + return strncpy(name, "eth0", IF_NAMESIZE); errno = ENXIO; return 0; } diff --git a/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c b/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c index 72ca21704e..6676a28831 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c @@ -4,11 +4,11 @@ struct if_nameindex *if_nameindex(void) { - /* Return a synthetic loopback interface */ + /* Return the kernel's synthetic virtual network interface. */ struct if_nameindex *idx = malloc(2 * sizeof(*idx)); if (!idx) return 0; idx[0].if_index = 1; - idx[0].if_name = strdup("lo"); + idx[0].if_name = strdup("eth0"); if (!idx[0].if_name) { free(idx); return 0; diff --git a/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c b/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c index 4c837e63ab..0141c18e31 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c @@ -3,6 +3,6 @@ unsigned if_nametoindex(const char *name) { - if (!strcmp(name, "lo")) return 1; + if (!strcmp(name, "eth0")) return 1; return 0; } diff --git a/programs/ifhwaddr.c b/programs/ifhwaddr.c index e30bab372b..8bf2072d44 100644 --- a/programs/ifhwaddr.c +++ b/programs/ifhwaddr.c @@ -33,6 +33,12 @@ int main(void) { for (i = 0; i < n; i++) { printf("name: %s\n", ifr[i].ifr_name); + unsigned index = if_nametoindex(ifr[i].ifr_name); + printf("index: %u\n", index); + if (index != 0) { + char name[IF_NAMESIZE]; + printf("index-name: %s\n", if_indextoname(index, name) ? name : "(null)"); + } if (ioctl(fd, SIOCGIFHWADDR, &ifr[i]) < 0) { perror("SIOCGIFHWADDR"); From c3ea58e21dc4376a8950c10e9d9e1160df49d680 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Wed, 17 Jun 2026 15:00:00 +0000 Subject: [PATCH 12/29] test: keep MemoryFS inode numbers representable --- host/test/vfs.test.ts | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/host/test/vfs.test.ts b/host/test/vfs.test.ts index 287204a509..95716122af 100644 --- a/host/test/vfs.test.ts +++ b/host/test/vfs.test.ts @@ -509,6 +509,42 @@ describe("MemoryFileSystem", () => { expect(entries).toContain("file.txt"); }); + it("reports raw inode numbers that remain representable after inode reuse", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_RDWR = 0x0002, + O_TRUNC = 0x0200; + + // SharedFS tracks an internal generation counter for reused inode slots. + // POSIX st_ino does not need to include that generation, and exposing it + // can overflow 32-bit guest language APIs while tools like ls(1) print the + // full kernel value. + for (let i = 0; i < 2_100; i++) { + const fd = mfs.open("/reuse.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + mfs.close(fd); + mfs.unlink("/reuse.txt"); + } + + const fd = mfs.open("/reuse.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const stat = mfs.fstat(fd); + expect(stat.ino).toBeGreaterThan(0); + expect(stat.ino).toBeLessThanOrEqual(0x7fffffff); + + const dh = mfs.opendir("/"); + let entry; + let dirIno: number | null = null; + while ((entry = mfs.readdir(dh)) !== null) { + if (entry.name === "reuse.txt") { + dirIno = entry.ino; + break; + } + } + mfs.closedir(dh); + expect(dirIno).toBe(stat.ino); + mfs.close(fd); + }); + it("honors O_CREAT|O_EXCL by failing when the final path already exists", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); From 5dcd4b00638c7e144f2e60df3b48082e86b1c5f7 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Thu, 18 Jun 2026 02:47:17 +0000 Subject: [PATCH 13/29] sharedfs: add indexed directory bookkeeping Track directory entries and allocation hints explicitly for large PHP test directories. This commit makes no performance claim; performance was not measured. --- host/src/vfs/sharedfs-vendor.ts | 667 ++++++++++++++++++++++++++------ 1 file changed, 545 insertions(+), 122 deletions(-) diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index 60b0eae721..4ec40bd740 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -109,7 +109,9 @@ const INO_INDIRECT = 88; const INO_DOUBLE_INDIRECT = 92; const INO_UID = 96; // u32 const INO_GID = 100; // u32 -// 104-127 reserved for future fields (flags, xattrs, etc.) +const INO_GENERATION = 104; // uint64, incremented when an inode slot is allocated +const INO_OPEN_COUNT = 112; // u32, open fd references +// 116-127 reserved for future fields (flags, xattrs, etc.) // FD entry layout const FD_INO = 4; @@ -125,6 +127,7 @@ const READER_MASK = 0x7fffffff | 0; export interface StatResult { ino: number; + generation: number; mode: number; linkCount: number; size: number; @@ -144,6 +147,20 @@ export interface SharedFsStats { maxName: number; } +interface DirIndexEntry { + ino: number; + abs: number; + recLen: number; + nameLen: number; +} + +interface DirIndex { + generation: number; + size: number; + entries: Map; + free: Array<{ abs: number; recLen: number }>; +} + const ERROR_MESSAGES: Record = { [ENOENT]: "No such file or directory", [EIO]: "I/O error", @@ -195,6 +212,18 @@ export class SharedFS { private view: DataView; private i32: Int32Array; private u8: Uint8Array; + private dirIndexes = new Map(); + private blockAllocHint = 0; + private inodeAllocHint = 2; + + /** + * Directory operations are stored in ext2-style variable-length entries. + * Linear scans are fine for normal directories, but workloads such as + * PHP's bug36365 test create tens of thousands of files in one directory. + * Build an in-memory name index once a directory reaches this size so + * creates/stat lookups stay near O(1) instead of O(n²). + */ + private static readonly DIR_INDEX_MIN_SIZE = 64 * 1024; private constructor(public readonly buffer: SharedArrayBuffer) { this.view = new DataView(buffer); @@ -278,16 +307,19 @@ export class SharedFS { const freeDataBlocks = totalBlocks - dataStart; Atomics.store(fs.i32, SB_FREE_BLOCKS >> 2, freeDataBlocks); + fs.blockAllocHint = dataStart; // Mark inodes 0 and 1 as used const ibStart = inodeBitmapStart * BLOCK_SIZE; fs.i32[ibStart >> 2] |= 0x3; Atomics.store(fs.i32, SB_FREE_INODES >> 2, totalInodes - 2); + fs.inodeAllocHint = 2; // Initialize root inode (inode 1) as empty directory const rootOff = fs.inodeOffset(ROOT_INO); fs.w32(rootOff + INO_MODE, S_IFDIR | 0o755); fs.w32(rootOff + INO_LINK_COUNT, 2); + fs.w64(rootOff + INO_GENERATION, 1); // Allocate a data block for root's directory entries const rootBlock = fs.blockAlloc(); @@ -327,6 +359,7 @@ export class SharedFS { throw new SFSError(EINVAL, "Bad version"); if (fs.r32(SB_BLOCK_SIZE) !== BLOCK_SIZE) throw new SFSError(EINVAL, "Bad block size"); + fs.resetAllocationHints(); return fs; } @@ -371,6 +404,34 @@ export class SharedFS { this.view.setBigUint64(off, BigInt(v), true); } + private resetAllocationHints(): void { + this.blockAllocHint = this.findNextFreeBlockHint(); + this.inodeAllocHint = this.findNextFreeInodeHint(); + } + + private findNextFreeBlockHint(): number { + const totalBlocks = this.r32(SB_TOTAL_BLOCKS); + const dataStart = this.r32(SB_DATA_START); + const bbStart = this.r32(SB_BLOCK_BITMAP_START) * BLOCK_SIZE; + for (let blockNo = dataStart; blockNo < totalBlocks; blockNo++) { + const idx = (bbStart >> 2) + (blockNo >> 5); + const bit = blockNo & 31; + if ((Atomics.load(this.i32, idx) & (1 << bit)) === 0) return blockNo; + } + return dataStart; + } + + private findNextFreeInodeHint(): number { + const totalInodes = this.r32(SB_TOTAL_INODES); + const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; + for (let ino = 2; ino < totalInodes; ino++) { + const idx = (ibStart >> 2) + (ino >> 5); + const bit = ino & 31; + if ((Atomics.load(this.i32, idx) & (1 << bit)) === 0) return ino; + } + return 2; + } + // ── Superblock lock (for grow) ─────────────────────────────────── private sbLock(): void { @@ -393,31 +454,34 @@ export class SharedFS { private blockAlloc(): number { const totalBlocks = this.r32(SB_TOTAL_BLOCKS); const bbStart = this.r32(SB_BLOCK_BITMAP_START) * BLOCK_SIZE; - const numWords = Math.ceil(totalBlocks / 32); - - for (let w = 0; w < numWords; w++) { - const idx = (bbStart >> 2) + w; + const dataStart = this.r32(SB_DATA_START); + const start = + this.blockAllocHint >= dataStart && this.blockAllocHint < totalBlocks + ? this.blockAllocHint + : dataStart; + const allocatableBlocks = totalBlocks - dataStart; + + for (let checked = 0; checked < allocatableBlocks; checked++) { + const blockNo = + dataStart + ((start - dataStart + checked) % allocatableBlocks); + const idx = (bbStart >> 2) + (blockNo >> 5); + const bit = blockNo & 31; const word = Atomics.load(this.i32, idx); - if (word === -1) continue; // all bits set (0xFFFFFFFF as int32) - - for (let bit = 0; bit < 32; bit++) { - const blockNo = w * 32 + bit; - if (blockNo >= totalBlocks) return ENOSPC; - if (word & (1 << bit)) continue; - - const desired = word | (1 << bit); - const old = Atomics.compareExchange(this.i32, idx, word, desired); - if (old === word) { - Atomics.sub(this.i32, SB_FREE_BLOCKS >> 2, 1); - // Zero the newly allocated block - const off = blockNo * BLOCK_SIZE; - this.u8.fill(0, off, off + BLOCK_SIZE); - return blockNo; - } - // CAS failed — retry this word - w--; - break; + if (word & (1 << bit)) continue; + + const desired = word | (1 << bit); + const old = Atomics.compareExchange(this.i32, idx, word, desired); + if (old === word) { + Atomics.sub(this.i32, SB_FREE_BLOCKS >> 2, 1); + this.blockAllocHint = + blockNo + 1 < totalBlocks ? blockNo + 1 : dataStart; + // Zero the newly allocated block + const off = blockNo * BLOCK_SIZE; + this.u8.fill(0, off, off + BLOCK_SIZE); + return blockNo; } + // CAS failed — retry this candidate. + checked--; } return ENOSPC; } @@ -443,6 +507,9 @@ export class SharedFS { if (old === word) break; } Atomics.add(this.i32, SB_FREE_BLOCKS >> 2, 1); + if (blockNo >= this.r32(SB_DATA_START) && blockNo < this.blockAllocHint) { + this.blockAllocHint = blockNo; + } } // ── Growth ─────────────────────────────────────────────────────── @@ -479,6 +546,7 @@ export class SharedFS { this.w32(SB_TOTAL_BLOCKS, newTotal); Atomics.add(this.i32, SB_FREE_BLOCKS >> 2, growBy); Atomics.add(this.i32, SB_GENERATION >> 2, 1); + this.blockAllocHint = current; return 0; } finally { this.sbUnlock(); @@ -497,34 +565,40 @@ export class SharedFS { private inodeAlloc(): number { const totalInodes = this.r32(SB_TOTAL_INODES); const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; - const numWords = Math.ceil(totalInodes / 32); - - for (let w = 0; w < numWords; w++) { - const idx = (ibStart >> 2) + w; + const start = + this.inodeAllocHint >= 2 && this.inodeAllocHint < totalInodes + ? this.inodeAllocHint + : 2; + const allocatableInodes = totalInodes - 2; + + for (let checked = 0; checked < allocatableInodes; checked++) { + const ino = 2 + ((start - 2 + checked) % allocatableInodes); + const idx = (ibStart >> 2) + (ino >> 5); + const bit = ino & 31; const word = Atomics.load(this.i32, idx); - if (word === -1) continue; - - for (let bit = 0; bit < 32; bit++) { - const ino = w * 32 + bit; - if (ino >= totalInodes) return ENOSPC; - if (word & (1 << bit)) continue; - - const desired = word | (1 << bit); - const old = Atomics.compareExchange(this.i32, idx, word, desired); - if (old === word) { - Atomics.sub(this.i32, SB_FREE_INODES >> 2, 1); - // Zero the inode - const off = this.inodeOffset(ino); - this.u8.fill(0, off, off + INODE_SIZE); - return ino; - } - w--; - break; + if (word & (1 << bit)) continue; + + const desired = word | (1 << bit); + const old = Atomics.compareExchange(this.i32, idx, word, desired); + if (old === word) { + Atomics.sub(this.i32, SB_FREE_INODES >> 2, 1); + this.inodeAllocHint = ino + 1 < totalInodes ? ino + 1 : 2; + // Zero the inode + const off = this.inodeOffset(ino); + this.u8.fill(0, off, off + INODE_SIZE); + this.w64(off + INO_GENERATION, this.nextInodeGeneration()); + return ino; } + // CAS failed — retry this candidate. + checked--; } return ENOSPC; } + private nextInodeGeneration(): number { + return Atomics.add(this.i32, SB_GENERATION >> 2, 1) + 1; + } + private inodeFree(ino: number): void { const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; const idx = (ibStart >> 2) + (ino >> 5); @@ -537,6 +611,61 @@ export class SharedFS { if (old === word) break; } Atomics.add(this.i32, SB_FREE_INODES >> 2, 1); + if (ino >= 2 && ino < this.inodeAllocHint) this.inodeAllocHint = ino; + } + + private inodeAddOpenRef(ino: number): void { + const idx = (this.inodeOffset(ino) + INO_OPEN_COUNT) >> 2; + Atomics.add(this.i32, idx, 1); + } + + private inodeDropOpenRef(ino: number): void { + let shouldFree = false; + this.inodeWriteLock(ino); + try { + const off = this.inodeOffset(ino); + const openCount = this.r32(off + INO_OPEN_COUNT); + if (openCount > 0) { + this.w32(off + INO_OPEN_COUNT, openCount - 1); + } + if (openCount <= 1 && this.r32(off + INO_LINK_COUNT) === 0) { + this.inodeTruncate(ino, 0); + shouldFree = true; + } + } finally { + this.inodeWriteUnlock(ino); + } + if (shouldFree) this.inodeFree(ino); + } + + private inodeDropLinkRefLocked(ino: number): boolean { + const off = this.inodeOffset(ino); + const linkCount = this.r32(off + INO_LINK_COUNT); + if (linkCount > 1) { + this.w32(off + INO_LINK_COUNT, linkCount - 1); + this.w64(off + INO_CTIME, Date.now()); + return false; + } + return this.inodeOrphanLocked(ino); + } + + private inodeOrphanLocked(ino: number): boolean { + const off = this.inodeOffset(ino); + this.w32(off + INO_LINK_COUNT, 0); + this.w64(off + INO_CTIME, Date.now()); + if (this.r32(off + INO_OPEN_COUNT) > 0) return false; + const mode = this.r32(off + INO_MODE); + const size = this.r64(off + INO_SIZE); + if ((mode & S_IFMT) === S_IFLNK && size <= INLINE_SYMLINK_SIZE) { + // Short symlink targets are stored inline in the inode's direct-pointer + // area. POSIX unlink removes the symlink inode itself even if the target + // is dangling; do not interpret inline target bytes as block numbers. + this.u8.fill(0, off + INO_DIRECT, off + INO_DIRECT + INLINE_SYMLINK_SIZE); + this.w64(off + INO_SIZE, 0); + } else { + this.inodeTruncate(ino, 0); + } + return true; } // ── Inode locking ──────────────────────────────────────────────── @@ -712,6 +841,11 @@ export class SharedFS { count: number, ): number { const inoOff = this.inodeOffset(ino); + const size = this.r64(inoOff + INO_SIZE); + if (offset > size) { + this.zeroInodeRange(ino, size, offset); + } + let totalWritten = 0; let srcPos = 0; @@ -733,8 +867,7 @@ export class SharedFS { totalWritten += chunk; } - const size = this.r64(inoOff + INO_SIZE); - if (offset > size) { + if (offset > this.r64(inoOff + INO_SIZE)) { this.w64(inoOff + INO_SIZE, offset); } if (totalWritten > 0) { @@ -745,6 +878,20 @@ export class SharedFS { return totalWritten; } + private zeroInodeRange(ino: number, start: number, end: number): void { + while (start < end) { + const fileBlock = Math.floor(start / BLOCK_SIZE); + const blockOff = start % BLOCK_SIZE; + const chunk = Math.min(BLOCK_SIZE - blockOff, end - start); + const phys = this.inodeBlockMap(ino, fileBlock, false); + if (phys > 0) { + const abs = phys * BLOCK_SIZE + blockOff; + this.u8.fill(0, abs, abs + chunk); + } + start += chunk; + } + } + private freeBlocksFrom(ino: number, fromBlock: number): void { const inoOff = this.inodeOffset(ino); @@ -816,6 +963,9 @@ export class SharedFS { const curSize = this.r64(inoOff + INO_SIZE); const sizeChanged = newSize !== curSize; if (newSize >= curSize) { + if (newSize > curSize) { + this.zeroInodeRange(ino, curSize, newSize); + } this.w64(inoOff + INO_SIZE, newSize); if (sizeChanged) { const now = Date.now(); @@ -824,6 +974,13 @@ export class SharedFS { } return; } + if (newSize % BLOCK_SIZE !== 0) { + this.zeroInodeRange( + ino, + newSize, + Math.ceil(newSize / BLOCK_SIZE) * BLOCK_SIZE, + ); + } const keepBlocks = Math.ceil(newSize / BLOCK_SIZE); this.freeBlocksFrom(ino, keepBlocks); this.w64(inoOff + INO_SIZE, newSize); @@ -836,9 +993,26 @@ export class SharedFS { // ── Directory operations ───────────────────────────────────────── - private dirLookup(dirIno: number, name: Uint8Array): number { - const inoOff = this.inodeOffset(dirIno); - const dirSize = this.r64(inoOff + INO_SIZE); + private dirNameKey(name: Uint8Array): string { + return safeDecode(name); + } + + private dirEntryNameMatches(abs: number, name: Uint8Array): boolean { + const entNameLen = this.view.getUint16(abs + 6, true); + if (entNameLen !== name.length) return false; + for (let i = 0; i < name.length; i++) { + if (this.u8[abs + DIRENT_HEADER_SIZE + i] !== name[i]) return false; + } + return true; + } + + private rebuildDirIndex( + dirIno: number, + generation: number, + dirSize: number, + ): DirIndex | number { + const entries = new Map(); + const free: Array<{ abs: number; recLen: number }> = []; let pos = 0; while (pos < dirSize) { @@ -860,37 +1034,159 @@ export class SharedFS { if (recLen === 0) return EIO; - if (entIno !== 0 && entNameLen === name.length) { - let match = true; - for (let i = 0; i < name.length; i++) { - if (this.u8[abs + DIRENT_HEADER_SIZE + i] !== name[i]) { - match = false; - break; - } - } - if (match) return entIno; + if (entIno !== 0) { + const name = safeDecode( + this.u8.subarray( + abs + DIRENT_HEADER_SIZE, + abs + DIRENT_HEADER_SIZE + entNameLen, + ), + ); + entries.set(name, { + ino: entIno, + abs, + recLen, + nameLen: entNameLen, + }); + } else if (recLen >= DIRENT_HEADER_SIZE) { + free.push({ abs, recLen }); } + off += recLen; } pos += remain; } - return ENOENT; + + const index = { generation, size: dirSize, entries, free }; + this.dirIndexes.set(dirIno, index); + return index; } - private dirAddEntry( + private getDirIndex(dirIno: number): DirIndex | null | number { + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + const generation = this.r64(inoOff + INO_GENERATION); + const cached = this.dirIndexes.get(dirIno); + if ( + cached && + cached.generation === generation && + cached.size === dirSize + ) { + return cached; + } + if (cached) this.dirIndexes.delete(dirIno); + + if (dirSize < SharedFS.DIR_INDEX_MIN_SIZE) return null; + return this.rebuildDirIndex(dirIno, generation, dirSize); + } + + private updateDirIndexAdd( dirIno: number, name: Uint8Array, childIno: number, - ): number { + abs: number, + recLen: number, + ): void { const inoOff = this.inodeOffset(dirIno); const dirSize = this.r64(inoOff + INO_SIZE); + const generation = this.r64(inoOff + INO_GENERATION); + const index = this.dirIndexes.get(dirIno); + if (!index) return; + if (index.generation !== generation) { + this.dirIndexes.delete(dirIno); + return; + } + index.size = dirSize; + index.entries.set(this.dirNameKey(name), { + ino: childIno, + abs, + recLen, + nameLen: name.length, + }); + } + + private useDirIndexFreeSlot( + index: DirIndex, + dirIno: number, + name: Uint8Array, + childIno: number, + ): boolean { const needed = align4(DIRENT_HEADER_SIZE + name.length); - // Track last entry position for potential recLen extension - let lastEntAbs = -1; + for (let i = index.free.length - 1; i >= 0; i--) { + const slot = index.free[i]; + if (slot.recLen < needed) continue; + index.free.splice(i, 1); + if ( + this.r32(slot.abs) !== 0 || + this.view.getUint16(slot.abs + 4, true) !== slot.recLen + ) { + continue; + } - // Scan existing entries for a deleted slot or slack space + this.w32(slot.abs, childIno); + this.view.setUint16(slot.abs + 6, name.length, true); + this.u8.set(name, slot.abs + DIRENT_HEADER_SIZE); + this.updateDirIndexAdd(dirIno, name, childIno, slot.abs, slot.recLen); + return true; + } + + return false; + } + + private updateDirIndexRemove(dirIno: number, name: Uint8Array): void { + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + const generation = this.r64(inoOff + INO_GENERATION); + const index = this.dirIndexes.get(dirIno); + if (!index) return; + if (index.generation !== generation || index.size !== dirSize) { + this.dirIndexes.delete(dirIno); + return; + } + index.entries.delete(this.dirNameKey(name)); + } + + private updateDirIndexRecLen( + dirIno: number, + abs: number, + recLen: number, + ): void { + const index = this.dirIndexes.get(dirIno); + if (!index) return; + for (const entry of index.entries.values()) { + if (entry.abs === abs) { + entry.recLen = recLen; + return; + } + } + } + + private dirLookup(dirIno: number, name: Uint8Array): number { + const index = this.getDirIndex(dirIno); + if (typeof index === "number") return index; + if (index) { + const entry = index.entries.get(this.dirNameKey(name)); + if (!entry) return ENOENT; + + // Validate positive hits against the backing directory entry so stale + // in-process indexes cannot resurrect an externally removed name. + if ( + this.r32(entry.abs) === entry.ino && + this.view.getUint16(entry.abs + 4, true) === entry.recLen && + this.view.getUint16(entry.abs + 6, true) === entry.nameLen && + this.dirEntryNameMatches(entry.abs, name) + ) { + return entry.ino; + } + + index.entries.delete(this.dirNameKey(name)); + return ENOENT; + } + + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); let pos = 0; + while (pos < dirSize) { const fileBlock = Math.floor(pos / BLOCK_SIZE); const blockOff = pos % BLOCK_SIZE; @@ -910,33 +1206,52 @@ export class SharedFS { if (recLen === 0) return EIO; - if (entIno === 0 && recLen >= needed) { - // Reuse deleted entry - this.w32(abs, childIno); - this.view.setUint16(abs + 6, name.length, true); - this.u8.set(name, abs + DIRENT_HEADER_SIZE); - return 0; - } - - // Check for slack space at end of this entry - const actualLen = align4(DIRENT_HEADER_SIZE + entNameLen); - const slack = recLen - actualLen; - if (entIno !== 0 && slack >= needed) { - // Shrink current entry, insert new one in slack - this.view.setUint16(abs + 4, actualLen, true); - const newAbs = abs + actualLen; - this.w32(newAbs, childIno); - this.view.setUint16(newAbs + 4, slack, true); - this.view.setUint16(newAbs + 6, name.length, true); - this.u8.set(name, newAbs + DIRENT_HEADER_SIZE); - return 0; + if (entIno !== 0 && entNameLen === name.length) { + let match = true; + for (let i = 0; i < name.length; i++) { + if (this.u8[abs + DIRENT_HEADER_SIZE + i] !== name[i]) { + match = false; + break; + } + } + if (match) return entIno; } - - lastEntAbs = abs; off += recLen; } pos += remain; } + return ENOENT; + } + + private findLastDirEntryInBlock( + dirIno: number, + fileBlock: number, + endOff: number, + ): number { + const phys = this.inodeBlockMap(dirIno, fileBlock, false); + if (phys <= 0) return -1; + const blockBase = phys * BLOCK_SIZE; + let off = 0; + let lastAbs = -1; + while (off < endOff) { + const abs = blockBase + off; + const recLen = this.view.getUint16(abs + 4, true); + if (recLen === 0 || off + recLen > endOff) return -1; + lastAbs = abs; + off += recLen; + } + return off === endOff ? lastAbs : -1; + } + + private dirAppendEntry( + dirIno: number, + name: Uint8Array, + childIno: number, + lastEntAbs = -1, + ): number { + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + const needed = align4(DIRENT_HEADER_SIZE + name.length); // No space found — append a new entry at the end. // Directory entries must not cross block boundaries (like ext2). @@ -956,10 +1271,21 @@ export class SharedFS { this.view.setUint16(padAbs + 4, gap, true); this.view.setUint16(padAbs + 6, 0, true); } - } else if (lastEntAbs >= 0) { - // Gap too small for a padding entry — extend last entry's recLen - const oldRecLen = this.view.getUint16(lastEntAbs + 4, true); - this.view.setUint16(lastEntAbs + 4, oldRecLen + gap, true); + } else { + if (lastEntAbs < 0) { + lastEntAbs = this.findLastDirEntryInBlock( + dirIno, + fileBlock, + blockOff, + ); + } + if (lastEntAbs >= 0) { + // Gap too small for a padding entry — extend last entry's recLen + const oldRecLen = this.view.getUint16(lastEntAbs + 4, true); + const newRecLen = oldRecLen + gap; + this.view.setUint16(lastEntAbs + 4, newRecLen, true); + this.updateDirIndexRecLen(dirIno, lastEntAbs, newRecLen); + } } appendPos = (fileBlock + 1) * BLOCK_SIZE; fileBlock++; @@ -983,10 +1309,111 @@ export class SharedFS { this.u8.set(name, abs + DIRENT_HEADER_SIZE); this.w64(inoOff + INO_SIZE, appendPos + needed); + this.updateDirIndexAdd(dirIno, name, childIno, abs, needed); return 0; } + private dirAddEntry( + dirIno: number, + name: Uint8Array, + childIno: number, + ): number { + const index = this.getDirIndex(dirIno); + if (typeof index === "number") return index; + if (index) { + if (this.useDirIndexFreeSlot(index, dirIno, name, childIno)) return 0; + + // Large indexed directories favor append-only growth when no indexed + // deleted slot is available. A full scan to discover slack on every + // create would reintroduce the O(n²) behavior the index avoids. + return this.dirAppendEntry(dirIno, name, childIno); + } + + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + const needed = align4(DIRENT_HEADER_SIZE + name.length); + + // Track last entry position for potential recLen extension + let lastEntAbs = -1; + + // Scan existing entries for a deleted slot or slack space + let pos = 0; + while (pos < dirSize) { + const fileBlock = Math.floor(pos / BLOCK_SIZE); + const blockOff = pos % BLOCK_SIZE; + const phys = this.inodeBlockMap(dirIno, fileBlock, false); + if (phys <= 0) return EIO; + + const blockBase = phys * BLOCK_SIZE; + let remain = dirSize - pos; + if (remain > BLOCK_SIZE - blockOff) remain = BLOCK_SIZE - blockOff; + + let off = blockOff; + while (off < blockOff + remain) { + const abs = blockBase + off; + const entIno = this.r32(abs); + const recLen = this.view.getUint16(abs + 4, true); + const entNameLen = this.view.getUint16(abs + 6, true); + + if (recLen === 0) return EIO; + + if (entIno === 0 && recLen >= needed) { + // Reuse deleted entry + this.w32(abs, childIno); + this.view.setUint16(abs + 6, name.length, true); + this.u8.set(name, abs + DIRENT_HEADER_SIZE); + this.updateDirIndexAdd(dirIno, name, childIno, abs, recLen); + return 0; + } + + // Check for slack space at end of this entry + const actualLen = align4(DIRENT_HEADER_SIZE + entNameLen); + const slack = recLen - actualLen; + if (entIno !== 0 && slack >= needed) { + // Shrink current entry, insert new one in slack + this.view.setUint16(abs + 4, actualLen, true); + const newAbs = abs + actualLen; + this.w32(newAbs, childIno); + this.view.setUint16(newAbs + 4, slack, true); + this.view.setUint16(newAbs + 6, name.length, true); + this.u8.set(name, newAbs + DIRENT_HEADER_SIZE); + this.updateDirIndexAdd(dirIno, name, childIno, newAbs, slack); + return 0; + } + + lastEntAbs = abs; + off += recLen; + } + pos += remain; + } + + return this.dirAppendEntry(dirIno, name, childIno, lastEntAbs); + } + private dirRemoveEntry(dirIno: number, name: Uint8Array): number { + const index = this.getDirIndex(dirIno); + if (typeof index === "number") return index; + if (index) { + const key = this.dirNameKey(name); + const entry = index.entries.get(key); + if (!entry) return ENOENT; + + if ( + this.r32(entry.abs) === entry.ino && + this.view.getUint16(entry.abs + 4, true) === entry.recLen && + this.view.getUint16(entry.abs + 6, true) === entry.nameLen && + this.dirEntryNameMatches(entry.abs, name) + ) { + this.w32(entry.abs, 0); // mark as deleted + index.entries.delete(key); + index.free.push({ abs: entry.abs, recLen: entry.recLen }); + return 0; + } + + index.entries.delete(key); + // Fall through to the linear scan below if the cached slot was stale. + } + const inoOff = this.inodeOffset(dirIno); const dirSize = this.r64(inoOff + INO_SIZE); let pos = 0; @@ -1020,6 +1447,7 @@ export class SharedFS { } if (match) { this.w32(abs, 0); // mark as deleted + this.updateDirIndexRemove(dirIno, name); return 0; } } @@ -1192,6 +1620,7 @@ export class SharedFS { this.w64(base + FD_OFFSET, 0); this.w32(base + FD_FLAGS, flags); this.w32(base + FD_IS_DIR, isDir ? 1 : 0); + this.inodeAddOpenRef(ino); return i; } } @@ -1233,6 +1662,7 @@ export class SharedFS { const off = this.inodeOffset(ino); return { ino, + generation: this.r64(off + INO_GENERATION), mode: this.r32(off + INO_MODE), linkCount: this.r32(off + INO_LINK_COUNT), size: this.r64(off + INO_SIZE), @@ -1333,6 +1763,7 @@ export class SharedFS { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); this.fdFree(fd); + this.inodeDropOpenRef(entry.ino); } read(fd: number, buffer: Uint8Array): number { @@ -1474,30 +1905,14 @@ export class SharedFS { const rc = this.dirRemoveEntry(parentIno, nameBytes); if (rc < 0) throw new SFSError(rc); + let shouldFree = false; this.inodeWriteLock(childIno); - const linkCount = this.r32(childOff + INO_LINK_COUNT); - if (linkCount <= 1) { - const size = this.r64(childOff + INO_SIZE); - if ((mode & S_IFMT) === S_IFLNK && size <= INLINE_SYMLINK_SIZE) { - // Short symlink targets live in the inode direct-pointer area. Unlink - // removes the symlink inode itself, not the inline target bytes as - // block pointers. - this.u8.fill( - 0, - childOff + INO_DIRECT, - childOff + INO_DIRECT + INLINE_SYMLINK_SIZE, - ); - this.w64(childOff + INO_SIZE, 0); - } else { - this.inodeTruncate(childIno, 0); - } - this.w32(childOff + INO_LINK_COUNT, 0); - this.inodeWriteUnlock(childIno); - this.inodeFree(childIno); - } else { - this.w32(childOff + INO_LINK_COUNT, linkCount - 1); + try { + shouldFree = this.inodeDropLinkRefLocked(childIno); + } finally { this.inodeWriteUnlock(childIno); } + if (shouldFree) this.inodeFree(childIno); } finally { this.inodeWriteUnlock(parentIno); } @@ -1524,15 +1939,21 @@ export class SharedFS { // Remove any existing entry at destination const existingIno = this.dirLookup(newParent, newNameBytes); if (existingIno >= 0) { + if (existingIno === srcIno) { + return; + } const existOff = this.inodeOffset(existingIno); const existMode = this.r32(existOff + INO_MODE); if ((existMode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); this.dirRemoveEntry(newParent, newNameBytes); + let shouldFreeExisting = false; this.inodeWriteLock(existingIno); - this.inodeTruncate(existingIno, 0); - this.w32(existOff + INO_LINK_COUNT, 0); - this.inodeWriteUnlock(existingIno); - this.inodeFree(existingIno); + try { + shouldFreeExisting = this.inodeDropLinkRefLocked(existingIno); + } finally { + this.inodeWriteUnlock(existingIno); + } + if (shouldFreeExisting) this.inodeFree(existingIno); } // Add entry in new directory @@ -1644,17 +2065,17 @@ export class SharedFS { const mode = this.r32(childOff + INO_MODE); if ((mode & S_IFMT) !== S_IFDIR) throw new SFSError(ENOTDIR); + let shouldFree = false; this.inodeWriteLock(childIno); try { if (!this.dirIsEmpty(childIno)) throw new SFSError(ENOTEMPTY); this.dirRemoveEntry(parentIno, nameBytes); - this.inodeTruncate(childIno, 0); - this.w32(childOff + INO_LINK_COUNT, 0); + shouldFree = this.inodeOrphanLocked(childIno); } finally { this.inodeWriteUnlock(childIno); } - this.inodeFree(childIno); + if (shouldFree) this.inodeFree(childIno); // Decrement parent link count const pOff = this.inodeOffset(parentIno); @@ -1716,7 +2137,8 @@ export class SharedFS { try { const off = this.inodeOffset(ino); const oldMode = this.r32(off + INO_MODE); - this.w32(off + INO_MODE, (oldMode & S_IFMT) | (mode & 0o7777)); + const fileType = (mode & S_IFMT) || (oldMode & S_IFMT); + this.w32(off + INO_MODE, fileType | (mode & 0o7777)); this.w64(off + INO_CTIME, Date.now()); } finally { this.inodeWriteUnlock(ino); @@ -1730,7 +2152,8 @@ export class SharedFS { try { const off = this.inodeOffset(entry.ino); const oldMode = this.r32(off + INO_MODE); - this.w32(off + INO_MODE, (oldMode & S_IFMT) | (mode & 0o7777)); + const fileType = (mode & S_IFMT) || (oldMode & S_IFMT); + this.w32(off + INO_MODE, fileType | (mode & 0o7777)); this.w64(off + INO_CTIME, Date.now()); } finally { this.inodeWriteUnlock(entry.ino); From e657398c8d9dcd4e333f34da25a5e8237711406a Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Thu, 18 Jun 2026 03:17:53 +0000 Subject: [PATCH 14/29] fix: align SharedFS directory reads and append offsets (cherry picked from commit 3c68749de2110373764ebd7788a4d7acce24f8fd) --- host/src/vfs/sharedfs-vendor.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index 4ec40bd740..69120981a6 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -1750,12 +1750,6 @@ export class SharedFS { const fd = this.fdAlloc(ino, flags, false); if (fd < 0) throw new SFSError(fd); - // If append, set offset to end - if (flags & O_APPEND) { - const base = FD_TABLE_OFFSET + fd * FD_ENTRY_SIZE; - this.w64(base + FD_OFFSET, this.r64(inoOff + INO_SIZE)); - } - return fd; } @@ -1769,6 +1763,9 @@ export class SharedFS { read(fd: number, buffer: Uint8Array): number { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); + const inoOff = this.inodeOffset(entry.ino); + const mode = this.r32(inoOff + INO_MODE); + if ((mode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); this.inodeReadLock(entry.ino); try { From fbfe975d45c151c7a63f4dfe0866518242fb2176 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Thu, 18 Jun 2026 04:07:20 +0000 Subject: [PATCH 15/29] fix: update SharedFS directory mutation times (cherry picked from commit 0449eb9afc05edaaa6e52e6364e597cbdd95fe66) --- host/src/vfs/sharedfs-vendor.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index 69120981a6..b3824a7e29 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -993,6 +993,13 @@ export class SharedFS { // ── Directory operations ───────────────────────────────────────── + private touchDirectoryMutation(dirIno: number): void { + const inoOff = this.inodeOffset(dirIno); + const now = Date.now(); + this.w64(inoOff + INO_MTIME, now); + this.w64(inoOff + INO_CTIME, now); + } + private dirNameKey(name: Uint8Array): string { return safeDecode(name); } @@ -1126,6 +1133,7 @@ export class SharedFS { this.w32(slot.abs, childIno); this.view.setUint16(slot.abs + 6, name.length, true); this.u8.set(name, slot.abs + DIRENT_HEADER_SIZE); + this.touchDirectoryMutation(dirIno); this.updateDirIndexAdd(dirIno, name, childIno, slot.abs, slot.recLen); return true; } @@ -1309,6 +1317,7 @@ export class SharedFS { this.u8.set(name, abs + DIRENT_HEADER_SIZE); this.w64(inoOff + INO_SIZE, appendPos + needed); + this.touchDirectoryMutation(dirIno); this.updateDirIndexAdd(dirIno, name, childIno, abs, needed); return 0; } @@ -1362,6 +1371,7 @@ export class SharedFS { this.w32(abs, childIno); this.view.setUint16(abs + 6, name.length, true); this.u8.set(name, abs + DIRENT_HEADER_SIZE); + this.touchDirectoryMutation(dirIno); this.updateDirIndexAdd(dirIno, name, childIno, abs, recLen); return 0; } @@ -1377,6 +1387,7 @@ export class SharedFS { this.view.setUint16(newAbs + 4, slack, true); this.view.setUint16(newAbs + 6, name.length, true); this.u8.set(name, newAbs + DIRENT_HEADER_SIZE); + this.touchDirectoryMutation(dirIno); this.updateDirIndexAdd(dirIno, name, childIno, newAbs, slack); return 0; } @@ -1407,6 +1418,7 @@ export class SharedFS { this.w32(entry.abs, 0); // mark as deleted index.entries.delete(key); index.free.push({ abs: entry.abs, recLen: entry.recLen }); + this.touchDirectoryMutation(dirIno); return 0; } @@ -1447,6 +1459,7 @@ export class SharedFS { } if (match) { this.w32(abs, 0); // mark as deleted + this.touchDirectoryMutation(dirIno); this.updateDirIndexRemove(dirIno, name); return 0; } From ae05120e5b33bb91e06d277ade8d19957a1a65d5 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Thu, 18 Jun 2026 05:12:21 +0000 Subject: [PATCH 16/29] fix: tighten SharedFS rename semantics (cherry picked from commit f0a021a9ff4fda5e0bd4d1762765975c10779f87) --- host/src/vfs/sharedfs-vendor.ts | 193 ++++++++++++++++++++++++++++---- host/test/vfs.test.ts | 93 +++++++++++++++ 2 files changed, 265 insertions(+), 21 deletions(-) diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index b3824a7e29..802e771646 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -190,6 +190,7 @@ export class SFSError extends Error { const encoder = new TextEncoder(); const decoder = new TextDecoder(); +const DOTDOT_BYTES = encoder.encode(".."); /** * Safely decode a Uint8Array that may be backed by SharedArrayBuffer. @@ -1471,6 +1472,77 @@ export class SharedFS { return ENOENT; } + private dirReplaceEntryIno( + dirIno: number, + name: Uint8Array, + childIno: number, + ): number { + const index = this.getDirIndex(dirIno); + if (typeof index === "number") return index; + if (index) { + const key = this.dirNameKey(name); + const entry = index.entries.get(key); + + if ( + entry && + this.r32(entry.abs) === entry.ino && + this.view.getUint16(entry.abs + 4, true) === entry.recLen && + this.view.getUint16(entry.abs + 6, true) === entry.nameLen && + this.dirEntryNameMatches(entry.abs, name) + ) { + this.w32(entry.abs, childIno); + entry.ino = childIno; + return 0; + } + + if (entry) index.entries.delete(key); + // Fall through to the linear scan below if the cached slot was stale. + } + + const inoOff = this.inodeOffset(dirIno); + const dirSize = this.r64(inoOff + INO_SIZE); + let pos = 0; + + while (pos < dirSize) { + const fileBlock = Math.floor(pos / BLOCK_SIZE); + const blockOff = pos % BLOCK_SIZE; + const phys = this.inodeBlockMap(dirIno, fileBlock, false); + if (phys <= 0) return EIO; + + const blockBase = phys * BLOCK_SIZE; + let remain = dirSize - pos; + if (remain > BLOCK_SIZE - blockOff) remain = BLOCK_SIZE - blockOff; + + let off = blockOff; + while (off < blockOff + remain) { + const abs = blockBase + off; + const entIno = this.r32(abs); + const recLen = this.view.getUint16(abs + 4, true); + const entNameLen = this.view.getUint16(abs + 6, true); + + if (recLen === 0) return EIO; + + if (entIno !== 0 && entNameLen === name.length) { + let match = true; + for (let i = 0; i < name.length; i++) { + if (this.u8[abs + DIRENT_HEADER_SIZE + i] !== name[i]) { + match = false; + break; + } + } + if (match) { + this.w32(abs, childIno); + this.updateDirIndexAdd(dirIno, name, childIno, abs, recLen); + return 0; + } + } + off += recLen; + } + pos += remain; + } + return ENOENT; + } + private dirIsEmpty(dirIno: number): boolean { const inoOff = this.inodeOffset(dirIno); const dirSize = this.r64(inoOff + INO_SIZE); @@ -1519,6 +1591,21 @@ export class SharedFS { return true; } + private dirIsAncestor(ancestorIno: number, dirIno: number): boolean { + let cur = dirIno; + + for (let depth = 0; depth < MAX_SYMLINK_HOPS * 1024; depth++) { + if (cur === ancestorIno) return true; + if (cur === ROOT_INO) return false; + + const parent = this.dirLookup(cur, DOTDOT_BYTES); + if (parent < 0 || parent === cur) return false; + cur = parent; + } + + return true; + } + // ── Path resolution ────────────────────────────────────────────── private pathResolve(path: string, followSymlinks: boolean): number { @@ -1935,6 +2022,8 @@ export class SharedFS { this.pathResolveParent(newPath); const oldNameBytes = encoder.encode(oldName); const newNameBytes = encoder.encode(newName); + const oldRequiresDirectory = oldPath.length > 1 && oldPath.endsWith("/"); + const newRequiresDirectory = newPath.length > 1 && newPath.endsWith("/"); // Lock both parents (consistent order to avoid deadlock) const first = Math.min(oldParent, newParent); @@ -1945,25 +2034,63 @@ export class SharedFS { try { const srcIno = this.dirLookup(oldParent, oldNameBytes); if (srcIno < 0) throw new SFSError(srcIno); + const srcOff = this.inodeOffset(srcIno); + const srcMode = this.r32(srcOff + INO_MODE); + const srcType = srcMode & S_IFMT; + + if ( + (oldRequiresDirectory || newRequiresDirectory) && + srcType !== S_IFDIR + ) { + throw new SFSError(ENOTDIR); + } + + if (srcType === S_IFDIR && this.dirIsAncestor(srcIno, newParent)) { + throw new SFSError(EINVAL); + } // Remove any existing entry at destination const existingIno = this.dirLookup(newParent, newNameBytes); + let removedExistingDirectory = false; if (existingIno >= 0) { if (existingIno === srcIno) { return; } const existOff = this.inodeOffset(existingIno); const existMode = this.r32(existOff + INO_MODE); - if ((existMode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); - this.dirRemoveEntry(newParent, newNameBytes); - let shouldFreeExisting = false; - this.inodeWriteLock(existingIno); - try { - shouldFreeExisting = this.inodeDropLinkRefLocked(existingIno); - } finally { - this.inodeWriteUnlock(existingIno); + const existType = existMode & S_IFMT; + + if (srcType === S_IFDIR) { + if (existType !== S_IFDIR) throw new SFSError(ENOTDIR); + + let shouldFreeExisting = false; + this.inodeWriteLock(existingIno); + try { + if (!this.dirIsEmpty(existingIno)) throw new SFSError(ENOTEMPTY); + const rc = this.dirRemoveEntry(newParent, newNameBytes); + if (rc < 0) throw new SFSError(rc); + shouldFreeExisting = this.inodeOrphanLocked(existingIno); + } finally { + this.inodeWriteUnlock(existingIno); + } + if (shouldFreeExisting) this.inodeFree(existingIno); + removedExistingDirectory = true; + } else { + if (existType === S_IFDIR) { + throw new SFSError(newRequiresDirectory ? ENOTDIR : EISDIR); + } + + const rc = this.dirRemoveEntry(newParent, newNameBytes); + if (rc < 0) throw new SFSError(rc); + let shouldFreeExisting = false; + this.inodeWriteLock(existingIno); + try { + shouldFreeExisting = this.inodeDropLinkRefLocked(existingIno); + } finally { + this.inodeWriteUnlock(existingIno); + } + if (shouldFreeExisting) this.inodeFree(existingIno); } - if (shouldFreeExisting) this.inodeFree(existingIno); } // Add entry in new directory @@ -1974,21 +2101,45 @@ export class SharedFS { this.dirRemoveEntry(oldParent, oldNameBytes); // Update link counts for directory renames - const srcOff = this.inodeOffset(srcIno); - const srcMode = this.r32(srcOff + INO_MODE); - if ( - (srcMode & S_IFMT) === S_IFDIR && - oldParent !== newParent - ) { - const oldPOff = this.inodeOffset(oldParent); - this.w32( - oldPOff + INO_LINK_COUNT, - this.r32(oldPOff + INO_LINK_COUNT) - 1, - ); + if (srcType === S_IFDIR) { + if (oldParent !== newParent) { + const oldPOff = this.inodeOffset(oldParent); + this.w32( + oldPOff + INO_LINK_COUNT, + this.r32(oldPOff + INO_LINK_COUNT) - 1, + ); + const newPOff = this.inodeOffset(newParent); + this.w32( + newPOff + INO_LINK_COUNT, + this.r32(newPOff + INO_LINK_COUNT) + 1, + ); + + this.inodeWriteLock(srcIno); + try { + const dotdotRc = this.dirReplaceEntryIno( + srcIno, + DOTDOT_BYTES, + newParent, + ); + if (dotdotRc < 0) throw new SFSError(dotdotRc); + this.w64(srcOff + INO_CTIME, Date.now()); + } finally { + this.inodeWriteUnlock(srcIno); + } + } + + if (removedExistingDirectory) { + const newPOff = this.inodeOffset(newParent); + this.w32( + newPOff + INO_LINK_COUNT, + this.r32(newPOff + INO_LINK_COUNT) - 1, + ); + } + } else if (removedExistingDirectory) { const newPOff = this.inodeOffset(newParent); this.w32( newPOff + INO_LINK_COUNT, - this.r32(newPOff + INO_LINK_COUNT) + 1, + this.r32(newPOff + INO_LINK_COUNT) - 1, ); } } finally { diff --git a/host/test/vfs.test.ts b/host/test/vfs.test.ts index 95716122af..eee49dbbc5 100644 --- a/host/test/vfs.test.ts +++ b/host/test/vfs.test.ts @@ -623,6 +623,99 @@ describe("MemoryFileSystem", () => { expect(() => mfs.stat("/todelete.txt")).toThrow(); }); + it("rejects rename source paths that require a non-directory to be a directory", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_WRONLY = 0x0001; + + const fd = mfs.open("/file.txt", O_CREAT | O_WRONLY, 0o644); + mfs.close(fd); + + expect(() => mfs.rename("/file.txt/", "/renamed.txt")).toThrow( + /Not a directory/, + ); + expect(mfs.stat("/file.txt").size).toBe(0); + expect(() => mfs.stat("/renamed.txt")).toThrow(/No such file/); + }); + + it("preserves POSIX type checks when renaming directories onto existing paths", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_WRONLY = 0x0001; + + mfs.mkdir("/dir", 0o755); + const fd = mfs.open("/file.txt", O_CREAT | O_WRONLY, 0o644); + mfs.close(fd); + mfs.symlink("/file.txt", "/link.txt"); + + expect(() => mfs.rename("/dir", "/file.txt")).toThrow(/Not a directory/); + expect(() => mfs.rename("/dir", "/link.txt")).toThrow(/Not a directory/); + + expect(mfs.stat("/dir").mode & 0xf000).toBe(0x4000); + expect(mfs.stat("/file.txt").mode & 0xf000).toBe(0x8000); + expect(mfs.readlink("/link.txt")).toBe("/file.txt"); + }); + + it("renames directories over empty directories and updates dot-dot", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_WRONLY = 0x0001; + + mfs.mkdir("/old-parent", 0o755); + mfs.mkdir("/new-parent", 0o755); + mfs.mkdir("/old-parent/child", 0o755); + const siblingFd = mfs.open( + "/new-parent/sibling.txt", + O_CREAT | O_WRONLY, + 0o644, + ); + mfs.close(siblingFd); + + mfs.rename("/old-parent/child", "/new-parent/child"); + expect(mfs.stat("/new-parent/child/../sibling.txt").mode & 0xf000).toBe( + 0x8000, + ); + + mfs.mkdir("/empty-dest", 0o755); + mfs.rename("/new-parent/child", "/empty-dest"); + expect(mfs.stat("/empty-dest").mode & 0xf000).toBe(0x4000); + expect(() => mfs.stat("/new-parent/child")).toThrow(/No such file/); + }); + + it("keeps an unlinked open file alive until close", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_RDWR = 0x0002, + O_TRUNC = 0x0200; + + const oldFd = mfs.open("/open.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const oldData = new TextEncoder().encode("old"); + mfs.write(oldFd, oldData, null, oldData.length); + mfs.unlink("/open.txt"); + expect(() => mfs.stat("/open.txt")).toThrow(); + + const newFd = mfs.open("/open.txt", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const newData = new TextEncoder().encode("newer"); + mfs.write(newFd, newData, null, newData.length); + + mfs.seek(oldFd, 0, 0); + const oldBuf = new Uint8Array(8); + const oldRead = mfs.read(oldFd, oldBuf, null, oldBuf.length); + expect(new TextDecoder().decode(oldBuf.subarray(0, oldRead))).toBe("old"); + + mfs.seek(newFd, 0, 0); + const newBuf = new Uint8Array(8); + const newRead = mfs.read(newFd, newBuf, null, newBuf.length); + expect(new TextDecoder().decode(newBuf.subarray(0, newRead))).toBe("newer"); + + mfs.close(oldFd); + mfs.close(newFd); + }); + it("ftruncate changes file size", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); From 32a5202f901bbb0a3297004183440264ee4a589e Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Thu, 18 Jun 2026 05:36:00 +0000 Subject: [PATCH 17/29] fix: reject trailing-slash unlink on files (cherry picked from commit f92b00daa3cf02af071da4cc9bc3887949e8ef85) --- host/src/vfs/sharedfs-vendor.ts | 4 ++++ host/test/vfs.test.ts | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index 802e771646..91cafad5be 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -1989,6 +1989,7 @@ export class SharedFS { unlink(path: string): void { const { parentIno, name } = this.pathResolveParent(path); const nameBytes = encoder.encode(name); + const requiresDirectory = path.length > 1 && path.endsWith("/"); this.inodeWriteLock(parentIno); try { @@ -1997,6 +1998,9 @@ export class SharedFS { const childOff = this.inodeOffset(childIno); const mode = this.r32(childOff + INO_MODE); + if (requiresDirectory && (mode & S_IFMT) !== S_IFDIR) { + throw new SFSError(ENOTDIR); + } if ((mode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); const rc = this.dirRemoveEntry(parentIno, nameBytes); diff --git a/host/test/vfs.test.ts b/host/test/vfs.test.ts index eee49dbbc5..f48b5fbce6 100644 --- a/host/test/vfs.test.ts +++ b/host/test/vfs.test.ts @@ -623,6 +623,22 @@ describe("MemoryFileSystem", () => { expect(() => mfs.stat("/todelete.txt")).toThrow(); }); + it("rejects unlink paths with a trailing slash on non-directories", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const O_CREAT = 0x0040, + O_WRONLY = 0x0001; + + const fd = mfs.open("/file.txt", O_CREAT | O_WRONLY, 0o644); + mfs.close(fd); + mfs.symlink("/file.txt", "/link.txt"); + + expect(() => mfs.unlink("/file.txt/")).toThrow(/Not a directory/); + expect(() => mfs.unlink("/link.txt/")).toThrow(/Not a directory/); + expect(mfs.stat("/file.txt").mode & 0xf000).toBe(0x8000); + expect(mfs.readlink("/link.txt")).toBe("/file.txt"); + }); + it("rejects rename source paths that require a non-directory to be a directory", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); From 7540902c44c6c30f147efedb58a3271e49e7073e Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Thu, 18 Jun 2026 06:24:00 +0000 Subject: [PATCH 18/29] fix: include service aliases in browser VFS (cherry picked from commit 374ff841ed8d8a0598b9f4a6526505db1b32763a) --- images/vfs/scripts/shell-vfs-build.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/images/vfs/scripts/shell-vfs-build.ts b/images/vfs/scripts/shell-vfs-build.ts index 969c554eb9..5136051214 100644 --- a/images/vfs/scripts/shell-vfs-build.ts +++ b/images/vfs/scripts/shell-vfs-build.ts @@ -175,8 +175,11 @@ function populateSystem(fs: MemoryFileSystem): void { "ssh\t\t22/tcp", "telnet\t\t23/tcp", "smtp\t\t25/tcp\t\tmail", + "nicname\t\t43/tcp\t\twhois", "domain\t\t53/tcp", "domain\t\t53/udp", + "gopher\t\t70/tcp", + "finger\t\t79/tcp", "http\t\t80/tcp\t\twww", "pop3\t\t110/tcp\t\tpop-3", "nntp\t\t119/tcp\t\treadnews untp", From d81b80d163377b41416dafa1c970832f32bc1828 Mon Sep 17 00:00:00 2001 From: Kandelo Agent Date: Thu, 18 Jun 2026 10:41:29 +0000 Subject: [PATCH 19/29] fix: arm browser VM interrupt timers from kernel workers Carry the cooperative VM interrupt hook into browser process workers so kernel-requested deadlines can interrupt CPU-bound guest execution. --- host/src/browser-kernel-worker-entry.ts | 56 ++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index 88f71b3bcb..b3c595d512 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -151,6 +151,11 @@ interface ProcessInfo { } const processes = new Map(); const processTeardowns = new Map>(); +interface VmInterruptTimer { + timer: ReturnType; + process: ProcessInfo; +} +const vmInterruptTimers = new Map(); // Includes standalone thread-worker teardown promises that may outlive the // process map entry they came from. const workerTeardowns = new Set>(); @@ -220,6 +225,44 @@ const threadWorkers = new Map(); const threadExits = new ThreadExitCoordinator(); const reportedNonzeroProcessExits = new Set(); +function clearVmInterruptTimer(pid: number): void { + const entry = vmInterruptTimers.get(pid); + if (entry) clearTimeout(entry.timer); + vmInterruptTimers.delete(pid); +} + +function handleVmInterruptTimer(msg: { + pid: number; + timedOutPtr: number; + vmInterruptPtr: number; + seconds: number; +}): void { + clearVmInterruptTimer(msg.pid); + if (!(msg.seconds > 0)) return; + const process = processes.get(msg.pid); + if (!process || !Number.isFinite(msg.seconds)) return; + const requestedDelayMs = msg.seconds * 1000; + // The process worker can be stuck in a CPU-bound Wasm loop, so a timer in + // that worker cannot set cooperative runtime interrupt flags. Run the timer + // from this kernel worker instead; the process memory is shared, matching + // the Node host's VM-interrupt timer path. + const delayMs = Math.min(0x7fffffff, Math.max(1, requestedDelayMs - 100)); + const timer = setTimeout(() => { + const active = vmInterruptTimers.get(msg.pid); + if (!active || active.timer !== timer) return; + vmInterruptTimers.delete(msg.pid); + if (processes.get(msg.pid) !== process) return; + const flags = new Uint8Array(process.memory.buffer); + if (Number.isSafeInteger(msg.timedOutPtr) && msg.timedOutPtr >= 0 && msg.timedOutPtr < flags.length) { + Atomics.store(flags, msg.timedOutPtr, 1); + } + if (Number.isSafeInteger(msg.vmInterruptPtr) && msg.vmInterruptPtr >= 0 && msg.vmInterruptPtr < flags.length) { + Atomics.store(flags, msg.vmInterruptPtr, 1); + } + }, delayMs); + vmInterruptTimers.set(msg.pid, { timer, process }); +} + function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -1031,7 +1074,7 @@ function installProcessWorkerListeners( worker.on("message", (msg: unknown) => { if (intentionallyTerminated.has(worker as object)) return; if (processes.get(pid)?.worker !== worker) return; - const m = msg as { type?: string; message?: string; pid?: number; status?: number }; + const m = msg as WorkerToHostMessage; if (m.type === "error") { console.error(`[kernel-worker] Process error pid=${pid}:`, m.message); // Forward to host stderr so the demo log shows the actual failure @@ -1048,6 +1091,9 @@ function installProcessWorkerListeners( finalize(classifiedTrapExitStatus(m.message) ?? -1, "worker-main error message", signum); } else if (m.type === "exit") { finalize(m.status ?? 0, "worker-main exit message"); + } else if (m.type === "vm_interrupt_timer") { + if (m.pid !== pid) return; + handleVmInterruptTimer(m); } }); } @@ -1194,6 +1240,7 @@ async function handleExec( try { const setupResult = kernelWorker.kernelExecSetup(pid, callerTid); if (setupResult < 0) return setupResult; + clearVmInterruptTimer(pid); // Invalidate the discarded image synchronously at the commit point. This // keeps stale clone/listener continuations out even if later detach or @@ -1575,6 +1622,9 @@ async function handleClone( // worker-main posted {type:"error"} — instantiation failure, top-level // throw, etc. Without this the parent's pthread_join blocks forever. failThread((m as { message?: string }).message ?? "thread error"); + } else if (m.type === "vm_interrupt_timer") { + if (!isCurrentThreadGeneration() || m.pid !== pid) return; + handleVmInterruptTimer(m); } }); threadWorker.on("error", (err: Error) => { @@ -1612,6 +1662,7 @@ async function finishProcessExit( const info = processes.get(pid); if (!info || info.worker !== expectedWorker) return; if (processTeardowns.has(expectedWorker)) return; + clearVmInterruptTimer(pid); reportNonzeroProcessExitDiagnostic(pid, exitStatus, "kernel process exit"); const threadedSettleMs = threadedProcessPids.has(pid) @@ -1672,6 +1723,7 @@ async function finishProcessExit( async function handleTerminateProcess(msg: Extract) { const pid = msg.pid; + clearVmInterruptTimer(pid); // Terminate thread workers const threads = threadWorkers.get(pid); @@ -1846,6 +1898,8 @@ async function handleDestroy(msg: Extract Date: Fri, 19 Jun 2026 08:11:47 -0400 Subject: [PATCH 20/29] fix: align clock, timer, and PHP runner behavior across hosts Validate supported clock and timer IDs, carry runtime hooks through Node and browser process launches, keep host time conversion bounded, and preserve PHP runner filesystem expectations. --- crates/kernel/src/syscalls.rs | 13 ++- crates/kernel/src/wasm_api.rs | 33 ++++++- crates/shared/src/lib.rs | 1 + docs/posix-status.md | 4 +- host/src/browser-kernel-worker-entry.ts | 61 ++++++++---- host/src/node-kernel-worker-entry.ts | 92 ++++++++++++++++++- host/src/platform/node.ts | 4 +- host/src/vfs/time.ts | 8 +- host/src/worker-main.ts | 58 +++++++++++- host/src/worker-protocol.ts | 11 ++- host/test/vfs.test.ts | 12 ++- packages/registry/php/test/browser/run-php.ts | 2 +- scripts/run-php-upstream-node-chunks.sh | 2 +- 13 files changed, 254 insertions(+), 47 deletions(-) diff --git a/crates/kernel/src/syscalls.rs b/crates/kernel/src/syscalls.rs index b1c9d67fc5..f11daff1d7 100644 --- a/crates/kernel/src/syscalls.rs +++ b/crates/kernel/src/syscalls.rs @@ -5576,12 +5576,11 @@ pub fn sys_nanosleep( pub fn sys_clock_getres(_proc: &Process, clock_id: u32) -> Result { use wasm_posix_shared::clock::*; match clock_id { - CLOCK_REALTIME | CLOCK_MONOTONIC | CLOCK_PROCESS_CPUTIME_ID | CLOCK_THREAD_CPUTIME_ID => { - Ok(WasmTimespec { - tv_sec: 0, - tv_nsec: 1_000_000, - }) // 1ms - } + CLOCK_REALTIME | CLOCK_MONOTONIC | CLOCK_PROCESS_CPUTIME_ID | CLOCK_THREAD_CPUTIME_ID + | CLOCK_BOOTTIME => Ok(WasmTimespec { + tv_sec: 0, + tv_nsec: 1_000_000, + }), // 1ms id if (id & 7) == 2 => { // Per-process CPU clock: clock_getcpuclockid encodes as (-pid-1)*8 + 2 Ok(WasmTimespec { @@ -5605,7 +5604,7 @@ pub fn sys_clock_nanosleep( ) -> Result<(), Errno> { use wasm_posix_shared::clock::*; // Validate clock_id - if clock_id != CLOCK_REALTIME && clock_id != CLOCK_MONOTONIC { + if clock_id != CLOCK_REALTIME && clock_id != CLOCK_MONOTONIC && clock_id != CLOCK_BOOTTIME { return Err(Errno::EINVAL); } // Validate timespec diff --git a/crates/kernel/src/wasm_api.rs b/crates/kernel/src/wasm_api.rs index 16583c446c..8af940bef9 100644 --- a/crates/kernel/src/wasm_api.rs +++ b/crates/kernel/src/wasm_api.rs @@ -9596,6 +9596,32 @@ pub extern "C" fn kernel_getitimer(which: u32, curr_ptr: *mut u8) -> i32 { /// TIMER_ABSTIME flag for timer_settime. const TIMER_ABSTIME: i32 = 1; +fn timer_clock_to_host_clock(clock_id: u32) -> Option { + use wasm_posix_shared::clock::*; + match clock_id { + CLOCK_REALTIME | CLOCK_MONOTONIC => Some(clock_id), + CLOCK_BOOTTIME => Some(CLOCK_MONOTONIC), + _ => None, + } +} + +#[cfg(test)] +mod posix_timer_tests { + use super::*; + use wasm_posix_shared::clock::*; + + #[test] + fn boottime_timers_use_monotonic_host_clock() { + assert_eq!(timer_clock_to_host_clock(CLOCK_BOOTTIME), Some(CLOCK_MONOTONIC)); + } + + #[test] + fn timer_create_rejects_unsupported_clock_ids() { + assert_eq!(timer_clock_to_host_clock(CLOCK_THREAD_CPUTIME_ID), None); + assert_eq!(timer_clock_to_host_clock(99), None); + } +} + /// timer_create(clock_id, sigevent_ptr, timerid_ptr) /// musl sends ksigevent = {sigev_value(i32), sigev_signo(i32), sigev_notify(i32), sigev_tid(i32)} = 16 bytes. /// Returns 0 on success, negative errno. @@ -9609,6 +9635,11 @@ pub extern "C" fn kernel_timer_create( let (_gkl, proc) = unsafe { get_process() }; + let host_clock_id = match timer_clock_to_host_clock(clock_id) { + Some(id) => id, + None => return -(Errno::EINVAL as i32), + }; + // Parse sigevent (default: SIGEV_SIGNAL with SIGALRM) let (sigev_signo, sigev_value, sigev_notify) = if sevp_ptr.is_null() { (14u32, 0i32, 0u32) // default: SIGALRM with SIGEV_SIGNAL @@ -9645,7 +9676,7 @@ pub extern "C" fn kernel_timer_create( }; proc.posix_timers[timer_id] = Some(PosixTimerState { - clock_id, + clock_id: host_clock_id, sigev_signo, sigev_value, interval_sec: 0, diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index 06f21ee513..aa888ffefc 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -897,6 +897,7 @@ pub mod clock { pub const CLOCK_MONOTONIC: u32 = 1; pub const CLOCK_PROCESS_CPUTIME_ID: u32 = 2; pub const CLOCK_THREAD_CPUTIME_ID: u32 = 3; + pub const CLOCK_BOOTTIME: u32 = 7; } /// Timespec structure for the Wasm POSIX interface. diff --git a/docs/posix-status.md b/docs/posix-status.md index 63fc4e13f7..f6f8134d25 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -259,7 +259,7 @@ shortcuts. |----------|--------|-------| | `time()` | Full | Wrapper around clock_gettime(CLOCK_REALTIME). Returns seconds since epoch. | | `gettimeofday()` | Full | Wrapper around clock_gettime(CLOCK_REALTIME). Returns (sec, usec) pair. | -| `clock_gettime()` | Full | Host-delegated. CLOCK_REALTIME and CLOCK_MONOTONIC supported. Node.js uses Date.now() and process.hrtime.bigint(). | +| `clock_gettime()` | Partial | Host-delegated. `CLOCK_REALTIME` and `CLOCK_MONOTONIC` are supported. `CLOCK_BOOTTIME` is a monotonic-equivalent fallback because Kandelo hosts do not expose suspend accounting. The process/thread CPU clocks currently report elapsed monotonic time rather than authoritative CPU usage. Node.js uses `Date.now()` and `process.hrtime.bigint()`; browsers use `Date.now()` and `performance.now()`. | | `nanosleep()` | Partial | Host-delegated. Node.js uses Atomics.wait with timeout. Browser support requires a worker context that can block with Atomics.wait. Validates tv_sec >= 0 and tv_nsec < 1e9. | | `usleep()` | Full | Converts microseconds to sec+nsec, delegates to host_nanosleep. | | `clock_settime()` | Stub | Returns EPERM. Cannot set system clock from Wasm. | @@ -293,7 +293,7 @@ shortcuts. | `inotify_init()` / `inotify_init1()` | Stub | Returns ENOSYS. | | `inotify_add_watch()` / `inotify_rm_watch()` | Stub | Returns EBADF. | | `fanotify_init()` / `fanotify_mark()` | Stub | Returns ENOSYS. | -| `timer_create()` | Partial | CLOCK_REALTIME and CLOCK_MONOTONIC with `SIGEV_SIGNAL` or `SIGEV_NONE`. `SIGEV_THREAD` and Linux-specific `SIGEV_THREAD_ID` return `ENOTSUP`. The timer syscall carries a 32-bit `sival_int`; wasm64 pointer-valued `sigev_value` is not supported. | +| `timer_create()` | Partial | `CLOCK_REALTIME`, `CLOCK_MONOTONIC`, and monotonic-equivalent `CLOCK_BOOTTIME`, with `SIGEV_SIGNAL` or `SIGEV_NONE`. `SIGEV_THREAD` and Linux-specific `SIGEV_THREAD_ID` return `ENOTSUP`. The timer syscall carries a 32-bit `sival_int`; wasm64 pointer-valued `sigev_value` is not supported. | | `timer_settime()` / `timer_gettime()` | Partial | Absolute (TIMER_ABSTIME) and relative timers and automatic interval rearming use host timers with millisecond granularity. `timer_gettime()` and `timer_settime()`'s old-value result currently report the last configured value rather than decreasing remaining time. | | `timer_getoverrun()` | Full | Tracks overrun count when signal is still pending at next interval fire. Reset on successful signal delivery. | | `timer_delete()` | Full | Cancels timer and removes from per-process table. | diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index b3c595d512..af5b8ac369 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -152,8 +152,11 @@ interface ProcessInfo { const processes = new Map(); const processTeardowns = new Map>(); interface VmInterruptTimer { - timer: ReturnType; + timer?: ReturnType; process: ProcessInfo; + deadlineMs: number; + timedOutPtr: number; + vmInterruptPtr: number; } const vmInterruptTimers = new Map(); // Includes standalone thread-worker teardown promises that may outlive the @@ -227,10 +230,26 @@ const reportedNonzeroProcessExits = new Set(); function clearVmInterruptTimer(pid: number): void { const entry = vmInterruptTimers.get(pid); - if (entry) clearTimeout(entry.timer); + if (entry?.timer !== undefined) clearTimeout(entry.timer); vmInterruptTimers.delete(pid); } +function scheduleVmInterruptTimer(pid: number, entry: VmInterruptTimer): void { + if (vmInterruptTimers.get(pid) !== entry || processes.get(pid) !== entry.process) return; + const remainingMs = entry.deadlineMs - performance.now(); + if (remainingMs <= 0) { + vmInterruptTimers.delete(pid); + const flags = new Uint8Array(entry.process.memory.buffer); + Atomics.store(flags, entry.timedOutPtr, 1); + Atomics.store(flags, entry.vmInterruptPtr, 1); + return; + } + entry.timer = setTimeout(() => { + entry.timer = undefined; + scheduleVmInterruptTimer(pid, entry); + }, Math.min(0x7fffffff, Math.max(1, remainingMs))); +} + function handleVmInterruptTimer(msg: { pid: number; timedOutPtr: number; @@ -241,26 +260,27 @@ function handleVmInterruptTimer(msg: { if (!(msg.seconds > 0)) return; const process = processes.get(msg.pid); if (!process || !Number.isFinite(msg.seconds)) return; - const requestedDelayMs = msg.seconds * 1000; + const flags = new Uint8Array(process.memory.buffer); + if ( + !Number.isSafeInteger(msg.timedOutPtr) || + msg.timedOutPtr < 0 || + msg.timedOutPtr >= flags.length || + !Number.isSafeInteger(msg.vmInterruptPtr) || + msg.vmInterruptPtr < 0 || + msg.vmInterruptPtr >= flags.length + ) return; // The process worker can be stuck in a CPU-bound Wasm loop, so a timer in // that worker cannot set cooperative runtime interrupt flags. Run the timer // from this kernel worker instead; the process memory is shared, matching // the Node host's VM-interrupt timer path. - const delayMs = Math.min(0x7fffffff, Math.max(1, requestedDelayMs - 100)); - const timer = setTimeout(() => { - const active = vmInterruptTimers.get(msg.pid); - if (!active || active.timer !== timer) return; - vmInterruptTimers.delete(msg.pid); - if (processes.get(msg.pid) !== process) return; - const flags = new Uint8Array(process.memory.buffer); - if (Number.isSafeInteger(msg.timedOutPtr) && msg.timedOutPtr >= 0 && msg.timedOutPtr < flags.length) { - Atomics.store(flags, msg.timedOutPtr, 1); - } - if (Number.isSafeInteger(msg.vmInterruptPtr) && msg.vmInterruptPtr >= 0 && msg.vmInterruptPtr < flags.length) { - Atomics.store(flags, msg.vmInterruptPtr, 1); - } - }, delayMs); - vmInterruptTimers.set(msg.pid, { timer, process }); + const entry: VmInterruptTimer = { + process, + deadlineMs: performance.now() + msg.seconds * 1000, + timedOutPtr: msg.timedOutPtr, + vmInterruptPtr: msg.vmInterruptPtr, + }; + vmInterruptTimers.set(msg.pid, entry); + scheduleVmInterruptTimer(msg.pid, entry); } function delay(ms: number): Promise { @@ -297,7 +317,6 @@ function reportNonzeroProcessExitDiagnostic( (serviceLog ? `\n${serviceLog}` : "") + `\n${syscalls}`; console.warn(diagnostic); - post({ type: "stderr", pid, data: new TextEncoder().encode(`${diagnostic}\n`) }); } function readServiceLogForProcess(argv: readonly string[] | undefined): string | null { @@ -1898,7 +1917,9 @@ async function handleDestroy(msg: Extract(); const processTeardowns = new Map>(); +interface VmInterruptTimer { + timer?: ReturnType; + process: ProcessInfo; + deadlineMs: number; + timedOutPtr: number; + vmInterruptPtr: number; +} +const vmInterruptTimers = new Map(); const reportedExits = new Set(); // Workers terminated by the kernel-worker entry itself (handleExit / @@ -196,6 +204,57 @@ function reportProcessExit(pid: number, status: number): void { post({ type: "exit", pid, status }); } +function clearVmInterruptTimer(pid: number): void { + const entry = vmInterruptTimers.get(pid); + if (entry?.timer !== undefined) clearTimeout(entry.timer); + vmInterruptTimers.delete(pid); +} + +function scheduleVmInterruptTimer(pid: number, entry: VmInterruptTimer): void { + if (vmInterruptTimers.get(pid) !== entry || processes.get(pid) !== entry.process) return; + const remainingMs = entry.deadlineMs - performance.now(); + if (remainingMs <= 0) { + vmInterruptTimers.delete(pid); + const flags = new Uint8Array(entry.process.memory.buffer); + Atomics.store(flags, entry.timedOutPtr, 1); + Atomics.store(flags, entry.vmInterruptPtr, 1); + return; + } + entry.timer = setTimeout(() => { + entry.timer = undefined; + scheduleVmInterruptTimer(pid, entry); + }, Math.min(0x7fffffff, Math.max(1, remainingMs))); +} + +function handleVmInterruptTimer(msg: { + pid: number; + timedOutPtr: number; + vmInterruptPtr: number; + seconds: number; +}, process: ProcessInfo): void { + if (processes.get(msg.pid) !== process) return; + clearVmInterruptTimer(msg.pid); + if (!(msg.seconds > 0)) return; + if (!Number.isFinite(msg.seconds)) return; + const flags = new Uint8Array(process.memory.buffer); + if ( + !Number.isSafeInteger(msg.timedOutPtr) || + msg.timedOutPtr < 0 || + msg.timedOutPtr >= flags.length || + !Number.isSafeInteger(msg.vmInterruptPtr) || + msg.vmInterruptPtr < 0 || + msg.vmInterruptPtr >= flags.length + ) return; + const entry: VmInterruptTimer = { + process, + deadlineMs: performance.now() + msg.seconds * 1000, + timedOutPtr: msg.timedOutPtr, + vmInterruptPtr: msg.vmInterruptPtr, + }; + vmInterruptTimers.set(msg.pid, entry); + scheduleVmInterruptTimer(msg.pid, entry); +} + function signalFromExitStatus(exitStatus: number): number | null { return exitStatus >= 128 ? (exitStatus - 128) & 0x7f : null; } @@ -233,6 +292,7 @@ async function finalizeProcessWorker( if (intentionallyTerminated.has(worker as object)) return; const cur = processes.get(pid); if (!cur || cur.worker !== worker) return; + clearVmInterruptTimer(pid); // Synthesize a signal-style reap *before* `deactivateProcess` in // case the worker died without sending SYS_EXIT_GROUP (uncaught @@ -770,7 +830,7 @@ function handleSpawn(msg: SpawnMessage) { // — so surface them to stderr and synthesize an exit so the host's // exitResolver fires with a non-zero status. worker.on("message", (raw: unknown) => { - const m = raw as { type: string; pid?: number; message?: string; status?: number }; + const m = raw as WorkerToHostMessage; if (m.type === "error" && m.pid === pid) { finalizeProcessWorkerError(pid, worker, m.message); } else if (m.type === "exit" && m.pid === pid) { @@ -779,6 +839,9 @@ function handleSpawn(msg: SpawnMessage) { // the kernel didn't process a SYS_exit_group first, the kernel // still has the process registered and host.spawn() would hang. void finalizeProcessWorker(pid, worker, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === pid) { + const process = processes.get(pid); + if (process?.worker === worker) handleVmInterruptTimer(m, process); } }); @@ -867,11 +930,14 @@ async function handleFork( childWorker.on("error", (err: Error) => finalizeUnexpectedWorkerError(childPid, childWorker, "worker error", err)); childWorker.on("message", (raw: unknown) => { - const m = raw as { type: string; pid?: number; message?: string; status?: number }; + const m = raw as WorkerToHostMessage; if (m.type === "error" && m.pid === childPid) { finalizeProcessWorkerError(childPid, childWorker, m.message); } else if (m.type === "exit" && m.pid === childPid) { void finalizeProcessWorker(childPid, childWorker, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === childPid) { + const process = processes.get(childPid); + if (process?.worker === childWorker) handleVmInterruptTimer(m, process); } }); @@ -934,6 +1000,7 @@ async function handleExec( try { const setupResult = kernelWorker.kernelExecSetup(pid, callerTid); if (setupResult < 0) return setupResult; + clearVmInterruptTimer(pid); // From this point onward the old image cannot resume. Invalidate its // channels and async continuations immediately, before any other @@ -1010,11 +1077,14 @@ async function handleExec( // uncaught wasm traps) so the host learns the process died — same // wiring as handleSpawn. replacementWorker.on("message", (raw: unknown) => { - const m = raw as { type: string; pid?: number; message?: string; status?: number }; + const m = raw as WorkerToHostMessage; if (m.type === "error" && m.pid === pid) { finalizeProcessWorkerError(pid, replacementWorker!, m.message); } else if (m.type === "exit" && m.pid === pid) { void finalizeProcessWorker(pid, replacementWorker!, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === pid) { + const process = processes.get(pid); + if (process?.worker === replacementWorker) handleVmInterruptTimer(m, process); } }); @@ -1163,11 +1233,14 @@ async function handlePosixSpawn( newWorker.on("error", (err: Error) => finalizeUnexpectedWorkerError(childPid, newWorker, "spawn worker error", err)); newWorker.on("message", (raw: unknown) => { - const m = raw as { type: string; pid?: number; message?: string; status?: number }; + const m = raw as WorkerToHostMessage; if (m.type === "error" && m.pid === childPid) { finalizeProcessWorkerError(childPid, newWorker, m.message); } else if (m.type === "exit" && m.pid === childPid) { void finalizeProcessWorker(childPid, newWorker, m.status ?? 0); + } else if (m.type === "vm_interrupt_timer" && m.pid === childPid) { + const process = processes.get(childPid); + if (process?.worker === newWorker) handleVmInterruptTimer(m, process); } }); @@ -1317,6 +1390,10 @@ async function handleClone( void terminateThreadEntry(); } else if (m.type === "error") { failThread(m.message); + } else if (m.type === "vm_interrupt_timer") { + if (isCurrentThreadGeneration() && m.pid === pid) { + handleVmInterruptTimer(m, processInfo); + } } }); threadWorker.on("error", (err: Error) => failThread(`worker error: ${err.message ?? err}`)); @@ -1340,6 +1417,7 @@ async function finishProcessExit( if (!expectedWorker) return; const info = processes.get(pid); if (!info || info.worker !== expectedWorker) return; + clearVmInterruptTimer(pid); const existingTeardown = processTeardowns.get(expectedWorker); if (existingTeardown) { @@ -1387,6 +1465,7 @@ async function finishProcessExit( async function handleTerminate(msg: TerminateProcessMessage) { const pid = msg.pid; + clearVmInterruptTimer(pid); // Terminate thread workers const threads = threadWorkers.get(pid); @@ -1423,6 +1502,7 @@ async function handleTerminate(msg: TerminateProcessMessage) { async function handleDestroy(msg: { requestId: number }) { const processEntries = [...processes.entries()]; for (const [pid, info] of processEntries) { + clearVmInterruptTimer(pid); await terminateThreadWorkers(pid); await terminateTrackedWorker(info.worker); try { kernelWorker.unregisterProcess(pid); } catch {} @@ -1438,6 +1518,10 @@ async function handleDestroy(msg: { requestId: number }) { } } processes.clear(); + for (const entry of vmInterruptTimers.values()) { + if (entry.timer !== undefined) clearTimeout(entry.timer); + } + vmInterruptTimers.clear(); processTeardowns.clear(); reportedExits.clear(); threadModuleCache.clear(); diff --git a/host/src/platform/node.ts b/host/src/platform/node.ts index f82f0bdab7..005eaa0263 100644 --- a/host/src/platform/node.ts +++ b/host/src/platform/node.ts @@ -299,8 +299,8 @@ export class NodePlatformIO implements PlatformIO { const elapsed = ns - this._startNs; return { sec: Number(elapsed / 1000000000n), nsec: Number(elapsed % 1000000000n) }; } - if (clockId === 1) { - // CLOCK_MONOTONIC + if (clockId === 1 || clockId === 7) { + // CLOCK_MONOTONIC / CLOCK_BOOTTIME return { sec: Number(ns / 1000000000n), nsec: Number(ns % 1000000000n) }; } // CLOCK_REALTIME — use hrtime + epoch offset for nanosecond resolution diff --git a/host/src/vfs/time.ts b/host/src/vfs/time.ts index 0136cd676b..4a9f82c6da 100644 --- a/host/src/vfs/time.ts +++ b/host/src/vfs/time.ts @@ -23,8 +23,8 @@ export class NodeTimeProvider implements TimeProvider { const elapsed = ns - this._startNs; return { sec: Number(elapsed / 1000000000n), nsec: Number(elapsed % 1000000000n) }; } - if (clockId === 1) { - // CLOCK_MONOTONIC + if (clockId === 1 || clockId === 7) { + // CLOCK_MONOTONIC / CLOCK_BOOTTIME return { sec: Number(ns / 1000000000n), nsec: Number(ns % 1000000000n) }; } // CLOCK_REALTIME — use hrtime + epoch offset for nanosecond resolution @@ -43,8 +43,8 @@ export class NodeTimeProvider implements TimeProvider { export class BrowserTimeProvider implements TimeProvider { clockGettime(clockId: number): { sec: number; nsec: number } { - if (clockId === 1 || clockId === 2 || clockId === 3) { - // CLOCK_MONOTONIC / CLOCK_PROCESS_CPUTIME_ID / CLOCK_THREAD_CPUTIME_ID + if (clockId === 1 || clockId === 2 || clockId === 3 || clockId === 7) { + // CLOCK_MONOTONIC / CPU-time clocks / CLOCK_BOOTTIME const ms = performance.now(); return { sec: Math.floor(ms / 1000), nsec: Math.floor((ms % 1000) * 1_000_000) }; } diff --git a/host/src/worker-main.ts b/host/src/worker-main.ts index 923443c49d..3a8f6c584d 100644 --- a/host/src/worker-main.ts +++ b/host/src/worker-main.ts @@ -668,6 +668,11 @@ function buildImportObject( dlopenImports?: Record, getInstance?: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8 = 4, + postVmInterruptTimer?: ( + timedOutPtr: number, + vmInterruptPtr: number, + seconds: number, + ) => void, ): WebAssembly.Imports { const envImports: Record = { memory }; /** Convert wasm64 BigInt pointer to number (safe since addresses < 4GB) */ @@ -702,6 +707,26 @@ function buildImportObject( Object.assign(envImports, dlopenImports); } + if ( + moduleImports.some( + (i) => + i.module === "env" && + i.name === "__wasm_posix_vm_interrupt_after" && + i.kind === "function", + ) + ) { + if (!postVmInterruptTimer) { + throw new Error("VM interrupt timer import requested without a host timer route"); + } + envImports.__wasm_posix_vm_interrupt_after = ( + timedOutPtr: number | bigint, + vmInterruptPtr: number | bigint, + seconds: number | bigint, + ): void => { + postVmInterruptTimer(n(timedOutPtr), n(vmInterruptPtr), n(seconds)); + }; + } + // C++ operator new/delete fallbacks — delegate to the wasm instance's malloc/free. // Normally resolved by MariaDB's my_new.cc (USE_MYSYS_NEW), but kept as safety net. if (getInstance) { @@ -1142,7 +1167,16 @@ export async function centralizedWorkerMain( hasDylinkForkRole, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth); + () => processInstance ?? undefined, ptrWidth, + (timedOutPtr, vmInterruptPtr, seconds) => { + port.postMessage({ + type: "vm_interrupt_timer", + pid, + timedOutPtr, + vmInterruptPtr, + seconds, + } satisfies WorkerToHostMessage); + }); const instance = await WebAssembly.instantiate(module, importObject); processInstance = instance; verifyProgramAbi(programBytes, initData.kernelAbiVersion, pid); @@ -1297,7 +1331,16 @@ export async function centralizedWorkerMain( false, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth); + () => processInstance ?? undefined, ptrWidth, + (timedOutPtr, vmInterruptPtr, seconds) => { + port.postMessage({ + type: "vm_interrupt_timer", + pid, + timedOutPtr, + vmInterruptPtr, + seconds, + } satisfies WorkerToHostMessage); + }); const instance = await WebAssembly.instantiate(module, importObject); processInstance = instance; verifyProgramAbi(programBytes, initData.kernelAbiVersion, pid); @@ -1993,7 +2036,16 @@ export async function centralizedThreadWorkerMain( }; } const importObject = buildImportObject(module, memory, kernelImports, channelOffset, undefined, - () => threadInstance, ptrWidth); + () => threadInstance, ptrWidth, + (timedOutPtr, vmInterruptPtr, seconds) => { + port.postMessage({ + type: "vm_interrupt_timer", + pid, + timedOutPtr, + vmInterruptPtr, + seconds, + } satisfies WorkerToHostMessage); + }); const instance = new WebAssembly.Instance(module, importObject); threadInstance = instance; diff --git a/host/src/worker-protocol.ts b/host/src/worker-protocol.ts index b7c4b127cd..c837a91ab2 100644 --- a/host/src/worker-protocol.ts +++ b/host/src/worker-protocol.ts @@ -99,7 +99,8 @@ export type WorkerToHostMessage = | WorkerErrorMessage | ExecRequestMessage | ExecCompleteMessage - | AlarmSetMessage; + | AlarmSetMessage + | VmInterruptTimerMessage; export interface WorkerReadyMessage { type: "ready"; @@ -141,6 +142,14 @@ export interface AlarmSetMessage { seconds: number; } +export interface VmInterruptTimerMessage { + type: "vm_interrupt_timer"; + pid: number; + timedOutPtr: number; + vmInterruptPtr: number; + seconds: number; +} + export interface ExecReplyMessage { type: "exec_reply"; wasmBytes: ArrayBuffer; diff --git a/host/test/vfs.test.ts b/host/test/vfs.test.ts index f48b5fbce6..ade23b0429 100644 --- a/host/test/vfs.test.ts +++ b/host/test/vfs.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { mkdtempSync, writeFileSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -933,4 +933,14 @@ describe("NodeTimeProvider", () => { const ns2 = BigInt(t2.sec) * 1_000_000_000n + BigInt(t2.nsec); expect(ns2).toBeGreaterThanOrEqual(ns1); }); + + it("treats CLOCK_BOOTTIME as monotonic-equivalent", () => { + const tp = new NodeTimeProvider(); + const monotonic = tp.clockGettime(1); + const boottime = tp.clockGettime(7); + const monotonicNs = BigInt(monotonic.sec) * 1_000_000_000n + BigInt(monotonic.nsec); + const boottimeNs = BigInt(boottime.sec) * 1_000_000_000n + BigInt(boottime.nsec); + expect(boottimeNs).toBeGreaterThanOrEqual(monotonicNs); + expect(boottimeNs - monotonicNs).toBeLessThan(100_000_000n); + }); }); diff --git a/packages/registry/php/test/browser/run-php.ts b/packages/registry/php/test/browser/run-php.ts index ac42a13db9..b8dedbedd4 100644 --- a/packages/registry/php/test/browser/run-php.ts +++ b/packages/registry/php/test/browser/run-php.ts @@ -43,7 +43,7 @@ async function runPhp( new SharedArrayBuffer(16 * 1024 * 1024, { maxByteLength: 64 * 1024 * 1024 }), 64 * 1024 * 1024, ); - for (const dir of ["/tmp", "/root", "/dev"]) ensureDir(memfs, dir); + for (const dir of ["/tmp", "/root", "/home", "/dev"]) ensureDir(memfs, dir); memfs.chmod("/tmp", 0o777); memfs.chmod("/root", 0o700); ensureDirRecursive(memfs, "/usr/local/bin"); diff --git a/scripts/run-php-upstream-node-chunks.sh b/scripts/run-php-upstream-node-chunks.sh index 19664df4c6..efab5bc5fb 100755 --- a/scripts/run-php-upstream-node-chunks.sh +++ b/scripts/run-php-upstream-node-chunks.sh @@ -269,7 +269,7 @@ while [ "$offset" -lt "$total" ]; do --timeout "$timeout_ms" \ --host-reset-interval "$host_reset_interval" \ --allow-unsupported \ - "${extra_args[@]}" \ + ${extra_args[@]+"${extra_args[@]}"} \ --json \ > "$jsonl" 2> "$stderr" status=$? From e8b6d31d305c78e3e8c0558688a2052c53826162 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 07:46:27 -0400 Subject: [PATCH 21/29] fix: preserve instrumented Wasm permissions in place --- crates/fork-instrument/src/main.rs | 29 ++++-- .../fork-instrument/tests/cli_permissions.rs | 95 +++++++++++++++++++ docs/fork-instrumentation.md | 4 + 3 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 crates/fork-instrument/tests/cli_permissions.rs diff --git a/crates/fork-instrument/src/main.rs b/crates/fork-instrument/src/main.rs index 29023af5f4..8f63f54983 100644 --- a/crates/fork-instrument/src/main.rs +++ b/crates/fork-instrument/src/main.rs @@ -67,34 +67,45 @@ fn main() -> Result<()> { let output_path = cli.output.as_ref().ok_or_else(|| { anyhow::anyhow!("--output is required unless --discover-only is set") })?; + // Capture this before writing: `--output` is allowed to name the input + // file, and output creation/truncation must not become the source of truth + // for the executable mode we are preserving. + let input_mode = input_mode(&cli.input)?; let output = instrument(&input, &opts) .with_context(|| format!("instrumenting {}", cli.input.display()))?; fs::write(output_path, &output) .with_context(|| format!("writing output: {}", output_path.display()))?; - preserve_input_permissions(&cli.input, output_path)?; + preserve_input_mode(input_mode, output_path)?; Ok(()) } #[cfg(unix)] -fn preserve_input_permissions(input_path: &Path, output_path: &Path) -> Result<()> { - let input_mode = fs::metadata(input_path) +fn input_mode(input_path: &Path) -> Result { + let mode = fs::metadata(input_path) .with_context(|| format!("stat input for permissions: {}", input_path.display()))? .permissions() .mode(); - let mut output_permissions = fs::metadata(output_path) - .with_context(|| format!("stat output for permissions: {}", output_path.display()))? - .permissions(); - output_permissions.set_mode(input_mode); - fs::set_permissions(output_path, output_permissions) + Ok(mode) +} + +#[cfg(not(unix))] +fn input_mode(_input_path: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn preserve_input_mode(input_mode: u32, output_path: &Path) -> Result<()> { + let permissions = fs::Permissions::from_mode(input_mode); + fs::set_permissions(output_path, permissions) .with_context(|| format!("setting output permissions: {}", output_path.display()))?; Ok(()) } #[cfg(not(unix))] -fn preserve_input_permissions(_input_path: &Path, _output_path: &Path) -> Result<()> { +fn preserve_input_mode(_input_mode: (), _output_path: &Path) -> Result<()> { Ok(()) } diff --git a/crates/fork-instrument/tests/cli_permissions.rs b/crates/fork-instrument/tests/cli_permissions.rs new file mode 100644 index 0000000000..5097e5d6c5 --- /dev/null +++ b/crates/fork-instrument/tests/cli_permissions.rs @@ -0,0 +1,95 @@ +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(0); + +const INPUT_WAT: &str = r#" +(module + (import "kernel" "kernel_fork" (func $fork (result i32))) + (memory 1) + (func (export "_start") + (drop (call $fork)))) +"#; + +struct TempDir(PathBuf); + +impl TempDir { + fn new(test_name: &str) -> Self { + let id = NEXT_TEMP_ID.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "kandelo-fork-instrument-{test_name}-{}-{id}", + std::process::id() + )); + fs::create_dir(&path).expect("create test directory"); + Self(path) + } + + fn join(&self, path: impl AsRef) -> PathBuf { + self.0.join(path) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn write_input(path: &Path, mode: u32) { + fs::write(path, wat::parse_str(INPUT_WAT).expect("compile input WAT")) + .expect("write input Wasm"); + fs::set_permissions(path, fs::Permissions::from_mode(mode)).expect("set input mode"); +} + +fn run_instrumenter(input: &Path, output: &Path) { + let status = Command::new(env!("CARGO_BIN_EXE_wasm-fork-instrument")) + .arg(input) + .arg("--output") + .arg(output) + .status() + .expect("run wasm-fork-instrument"); + assert!(status.success(), "instrumenter exited with {status}"); +} + +fn mode(path: &Path) -> u32 { + fs::metadata(path) + .expect("stat output") + .permissions() + .mode() + & 0o7777 +} + +#[test] +fn new_output_preserves_input_mode() { + let dir = TempDir::new("new-output"); + let input = dir.join("input.wasm"); + let output = dir.join("output.wasm"); + write_input(&input, 0o751); + + run_instrumenter(&input, &output); + + assert_eq!(mode(&input), 0o751); + assert_eq!(mode(&output), 0o751); + wasmparser::Validator::new() + .validate_all(&fs::read(output).expect("read output")) + .expect("instrumented output validates"); +} + +#[test] +fn in_place_output_preserves_input_mode() { + let dir = TempDir::new("in-place"); + let input = dir.join("program.wasm"); + write_input(&input, 0o711); + + run_instrumenter(&input, &input); + + assert_eq!(mode(&input), 0o711); + wasmparser::Validator::new() + .validate_all(&fs::read(input).expect("read output")) + .expect("instrumented output validates"); +} diff --git a/docs/fork-instrumentation.md b/docs/fork-instrumentation.md index 98c5350dee..f45344b322 100644 --- a/docs/fork-instrumentation.md +++ b/docs/fork-instrumentation.md @@ -38,6 +38,10 @@ ABI version: `12` (see - Do not keep compiler/linker flags solely for the retired legacy path. The fork instrumenter does not require preserved function names or onlylists; if a build keeps debug-info flags, it should be for a current diagnostic reason. +- On Unix hosts, the CLI preserves the input Wasm file's permission mode on + its output, including when `--output` names the input file. Package build + scripts can therefore instrument installed executables in place without + making them non-executable. ## State machine From 085a027d0ea27526fcd0d2ed5ebb72fb56537c35 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 07:47:27 -0400 Subject: [PATCH 22/29] fix: align Node host mounts and worker resources Resolve host-backed paths component by component, preserve virtual ownership and native timestamp identity, and keep mount operations inside their configured root. Validate the larger Node worker stack policy with real recursive Wasm workers. --- docs/software-unit-tests.md | 8 +- host/src/platform/native-metadata.ts | 51 +++++- host/src/platform/node.ts | 5 +- host/src/vfs/host-fs.ts | 141 ++++++++-------- host/src/worker-adapter.ts | 12 +- .../fixtures/deep-wasm-recursion-worker.mjs | 7 + host/test/fixtures/deep-wasm-recursion.wat | 14 ++ host/test/global-setup.ts | 8 +- host/test/vfs/host-fs-path-resolution.test.ts | 157 ++++++++++++++++++ host/test/worker-adapter.test.ts | 80 ++++++++- scripts/run-php-upstream-tests.ts | 6 + 11 files changed, 408 insertions(+), 81 deletions(-) create mode 100644 host/test/fixtures/deep-wasm-recursion-worker.mjs create mode 100644 host/test/fixtures/deep-wasm-recursion.wat create mode 100644 host/test/vfs/host-fs-path-resolution.test.ts diff --git a/docs/software-unit-tests.md b/docs/software-unit-tests.md index 11f8a1bfe4..8637aaa8a8 100644 --- a/docs/software-unit-tests.md +++ b/docs/software-unit-tests.md @@ -64,8 +64,12 @@ scripts/run-php-upstream-node-chunks.sh \ Important reruns included: - `Zend/tests/generators/bug71441.phpt`, which now passes after increasing the - default Node host worker stack from 16 MiB to 32 MiB. This is a general host - stability change for deep guest stacks, not PHP-specific behavior. + Node process-worker stack limit to 32 MiB. This is a general host stability + change for deep guest stacks, not PHP-specific behavior. Embedders with a + tighter memory budget can set `KANDELO_NODE_WORKER_STACK_SIZE_MB` to a + positive finite value; the browser harness instead launches Chromium with + an equivalent V8 stack limit because browser workers do not expose Node's + per-worker `resourceLimits` API. - FPM non-root/virtual-ownership coverage, which now passes after allowing host-backed mounts to expose stable virtual uid/gid metadata to the guest. - Two online `httpbin.org` HTTP/1.1 PHPTs, which are now counted as upstream diff --git a/host/src/platform/native-metadata.ts b/host/src/platform/native-metadata.ts index bdc904a26b..ef1954ddd7 100644 --- a/host/src/platform/native-metadata.ts +++ b/host/src/platform/native-metadata.ts @@ -13,6 +13,9 @@ interface VirtualMetadata { gid?: number; atimeMs?: number; mtimeMs?: number; + nativeAtimeMs?: number; + nativeMtimeMs?: number; + nativeCtimeMs?: number; ctimeMs?: number; } @@ -33,6 +36,7 @@ export class NativeMetadataOverlay { toStatResult(s: Stats): StatResult { const metadata = this.entries.get(this.key(s)); + if (metadata !== undefined) this.reconcileNativeTimes(metadata, s); return { dev: s.dev, ino: s.ino, @@ -64,18 +68,25 @@ export class NativeMetadataOverlay { metadata.ctimeMs = Date.now(); } - utimens(s: Stats, atimeMs: number, mtimeMs: number, ctimeMs = Date.now()): void { + utimens( + s: Stats, + atimeMs: number, + mtimeMs: number, + nativeAfter: Stats, + ): void { const metadata = this.metadataFor(s); metadata.atimeMs = atimeMs; metadata.mtimeMs = mtimeMs; - metadata.ctimeMs = Math.max(metadata.ctimeMs ?? 0, ctimeMs); + metadata.nativeAtimeMs = nativeAfter.atimeMs; + metadata.nativeMtimeMs = nativeAfter.mtimeMs; + metadata.nativeCtimeMs = nativeAfter.ctimeMs; + metadata.ctimeMs = Math.max(metadata.ctimeMs ?? 0, nativeAfter.ctimeMs); } noteNativeContentChange(s: Stats): void { const metadata = this.entries.get(this.key(s)); if (metadata === undefined) return; - delete metadata.atimeMs; - delete metadata.mtimeMs; + this.clearTimeOverrides(metadata); } forget(s: Stats): void { @@ -99,6 +110,38 @@ export class NativeMetadataOverlay { return metadata; } + private reconcileNativeTimes(metadata: VirtualMetadata, s: Stats): void { + if (metadata.nativeCtimeMs === undefined) return; + + const nativeMetadataChanged = s.ctimeMs !== metadata.nativeCtimeMs; + if ( + nativeMetadataChanged || + (metadata.nativeAtimeMs !== undefined && s.atimeMs !== metadata.nativeAtimeMs) + ) { + delete metadata.atimeMs; + delete metadata.nativeAtimeMs; + } + if ( + nativeMetadataChanged || + (metadata.nativeMtimeMs !== undefined && s.mtimeMs !== metadata.nativeMtimeMs) + ) { + delete metadata.mtimeMs; + delete metadata.nativeMtimeMs; + } + + if (metadata.atimeMs === undefined && metadata.mtimeMs === undefined) { + delete metadata.nativeCtimeMs; + } + } + + private clearTimeOverrides(metadata: VirtualMetadata): void { + delete metadata.atimeMs; + delete metadata.mtimeMs; + delete metadata.nativeAtimeMs; + delete metadata.nativeMtimeMs; + delete metadata.nativeCtimeMs; + } + private key(s: Stats): string { return `${s.dev}:${s.ino}`; } diff --git a/host/src/platform/node.ts b/host/src/platform/node.ts index 005eaa0263..8c9c4a2738 100644 --- a/host/src/platform/node.ts +++ b/host/src/platform/node.ts @@ -13,6 +13,9 @@ import type { PlatformIO, StatResult, StatfsResult } from "../types"; import { nativeStatfs, translateOpenFlags } from "../vfs/host-fs"; import { NativeMetadataOverlay } from "./native-metadata"; +const UTIME_NOW = 0x3fffffff; +const UTIME_OMIT = 0x3ffffffe; + export class NodePlatformIO implements PlatformIO { private dirHandles = new Map(); private nextDirHandle = 1; @@ -236,7 +239,7 @@ export class NodePlatformIO implements PlatformIO { ? nowMs : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); fs.utimesSync(nativePath, atimeMs / 1000, mtimeMs / 1000); - this.metadata.utimens(stat, atimeMs, mtimeMs, fs.statSync(nativePath).ctimeMs); + this.metadata.utimens(stat, atimeMs, mtimeMs, fs.statSync(nativePath)); } opendir(path: string): number { diff --git a/host/src/vfs/host-fs.ts b/host/src/vfs/host-fs.ts index 32427bbd2d..79d12221d3 100644 --- a/host/src/vfs/host-fs.ts +++ b/host/src/vfs/host-fs.ts @@ -45,11 +45,11 @@ export function translateOpenFlags(linuxFlags: number): number { if (linuxFlags & L_O_TRUNC) native |= fs.constants.O_TRUNC; if (linuxFlags & L_O_APPEND) native |= fs.constants.O_APPEND; if (linuxFlags & L_O_NONBLOCK) native |= fs.constants.O_NONBLOCK; - if ((linuxFlags & L_O_DIRECTORY) && fs.constants.O_DIRECTORY) + if (linuxFlags & L_O_DIRECTORY && fs.constants.O_DIRECTORY) native |= fs.constants.O_DIRECTORY; - if ((linuxFlags & L_O_NOFOLLOW) && fs.constants.O_NOFOLLOW) + if (linuxFlags & L_O_NOFOLLOW && fs.constants.O_NOFOLLOW) native |= fs.constants.O_NOFOLLOW; - if ((linuxFlags & L_O_NOCTTY) && fs.constants.O_NOCTTY) + if (linuxFlags & L_O_NOCTTY && fs.constants.O_NOCTTY) native |= fs.constants.O_NOCTTY; // O_LARGEFILE and O_CLOEXEC have no Node.js equivalent; ignored. @@ -91,8 +91,6 @@ export class HostFileSystem implements FileSystemBackend { private dirHandles = new Map(); private nextDirHandle = 1; private metadata: NativeMetadataOverlay; - private dirPathCache = new Map(); - private readonly maxDirPathCacheEntries = 4096; constructor( rootPath: string, @@ -104,7 +102,10 @@ export class HostFileSystem implements FileSystemBackend { ? fs.realpathSync(resolvedRoot) : resolvedRoot; this.guestMountPoint = this.normalizeGuestMountPoint(guestMountPoint); - this.metadata = new NativeMetadataOverlay(options.uid ?? 0, options.gid ?? 0); + this.metadata = new NativeMetadataOverlay( + options.uid ?? 0, + options.gid ?? 0, + ); } /** @@ -118,6 +119,11 @@ export class HostFileSystem implements FileSystemBackend { * up as a directory first. Lexical normalization would incorrectly collapse * that to `existing/file`. * + * Resolved prefixes are deliberately not cached. A host-backed tree can be + * changed externally or through another mount of the same directory; using + * a stale prefix after a directory-to-symlink replacement would bypass the + * component checks below. + * * Native symlink targets are stored as guest strings. When following a * symlink whose target is absolute and still inside this mount, translate it * back to a mount-relative path before continuing. This preserves readlink(2) @@ -125,34 +131,18 @@ export class HostFileSystem implements FileSystemBackend { */ private safePath(relative: string, followFinal = true): string { const hadTrailingSlash = relative.length > 1 && /\/+$/.test(relative); - const originalParts = this.pathParts(relative); let current = this.rootPath; - let pending = [...originalParts]; - let processed: string[] = []; + let pending = this.pathParts(relative); let symlinkDepth = 0; - let cacheable = !originalParts.includes(".."); - - if (cacheable) { - for (let i = originalParts.length; i > 0; i--) { - const cached = this.dirPathCache.get(originalParts.slice(0, i).join("/")); - if (cached === undefined) continue; - current = cached; - pending = originalParts.slice(i); - processed = originalParts.slice(0, i); - break; - } - } while (pending.length > 0) { const part = pending.shift()!; if (part === ".") continue; if (part === "..") { - cacheable = false; if (current === this.rootPath) { throw new Error("EACCES: path traversal blocked"); } current = nodePath.dirname(current); - processed.pop(); continue; } @@ -172,8 +162,8 @@ export class HostFileSystem implements FileSystemBackend { } if (shouldFollow && lst.isSymbolicLink()) { - cacheable = false; - if (++symlinkDepth > 40) throw new Error("ELOOP: too many symbolic links"); + if (++symlinkDepth > 40) + throw new Error("ELOOP: too many symbolic links"); const target = fs.readlinkSync(candidate, "utf8"); if (target.startsWith("/")) { const mountRelative = this.guestAbsoluteToMountRelative(target); @@ -195,14 +185,16 @@ export class HostFileSystem implements FileSystemBackend { if (!isFinal) { current = fs.realpathSync(candidate); this.assertWithinRoot(current); - processed.push(part); - if (cacheable) this.setCachedDirPath(processed, current); } else { current = candidate; } } - if (hadTrailingSlash && current !== this.rootPath && !current.endsWith(nodePath.sep)) { + if ( + hadTrailingSlash && + current !== this.rootPath && + !current.endsWith(nodePath.sep) + ) { // Keep a final separator for native fs calls. POSIX requires a // trailing slash to resolve the preceding component as a directory; the // native call then returns ENOTDIR for regular files while still @@ -213,19 +205,6 @@ export class HostFileSystem implements FileSystemBackend { return current; } - private setCachedDirPath(parts: string[], nativePath: string): void { - if (parts.length === 0) return; - this.dirPathCache.set(parts.join("/"), nativePath); - if (this.dirPathCache.size > this.maxDirPathCacheEntries) { - const oldest = this.dirPathCache.keys().next().value; - if (oldest !== undefined) this.dirPathCache.delete(oldest); - } - } - - private clearDirPathCache(): void { - this.dirPathCache.clear(); - } - private normalizeGuestMountPoint(mountPoint: string): string { if (!mountPoint.startsWith("/")) mountPoint = `/${mountPoint}`; return mountPoint !== "/" && mountPoint.endsWith("/") @@ -252,7 +231,11 @@ export class HostFileSystem implements FileSystemBackend { private assertWithinRoot(path: string): void { const rel = nodePath.relative(this.rootPath, path); if (rel === "") return; - if (rel.startsWith("..") || nodePath.isAbsolute(rel)) { + if ( + rel === ".." || + rel.startsWith(`..${nodePath.sep}`) || + nodePath.isAbsolute(rel) + ) { throw new Error("EACCES: path traversal blocked"); } } @@ -271,7 +254,10 @@ export class HostFileSystem implements FileSystemBackend { // ── File handle operations ─────────────────────────────────── open(path: string, flags: number, mode: number): number { - const nativePath = this.safePath(path, (flags & 0o400000) === 0); + const noFollowFinal = + (flags & 0o400000) !== 0 || + ((flags & 0o100) !== 0 && (flags & 0o200) !== 0); + const nativePath = this.safePath(path, !noFollowFinal); const created = (flags & 0o100) !== 0 && !fs.existsSync(nativePath); const fd = fs.openSync(nativePath, translateOpenFlags(flags), mode); if (created) this.metadata.chmod(fs.fstatSync(fd), mode); @@ -307,7 +293,8 @@ export class HostFileSystem implements FileSystemBackend { ): number { const pos = offset ?? this.fdPositions.get(handle) ?? 0; const bytesWritten = fs.writeSync(handle, buffer, 0, length, pos); - if (bytesWritten > 0) this.metadata.noteNativeContentChange(fs.fstatSync(handle)); + if (bytesWritten > 0) + this.metadata.noteNativeContentChange(fs.fstatSync(handle)); if (offset === null) { this.fdPositions.set(handle, pos + bytesWritten); } @@ -369,7 +356,7 @@ export class HostFileSystem implements FileSystemBackend { } mkdir(path: string, mode: number): void { - const nativePath = this.safePath(path); + const nativePath = this.safePath(path, false); fs.mkdirSync(nativePath, { mode }); this.metadata.chmod(fs.statSync(nativePath), mode); } @@ -378,7 +365,6 @@ export class HostFileSystem implements FileSystemBackend { const nativePath = this.safePath(path, false); const stat = fs.lstatSync(nativePath); fs.rmdirSync(nativePath); - this.clearDirPathCache(); this.metadata.forget(stat); } @@ -386,7 +372,6 @@ export class HostFileSystem implements FileSystemBackend { const nativePath = this.safePath(path, false); const stat = fs.lstatSync(nativePath); fs.unlinkSync(nativePath); - if (stat.isSymbolicLink()) this.clearDirPathCache(); if (stat.nlink <= 1) this.metadata.forget(stat); } @@ -397,12 +382,18 @@ export class HostFileSystem implements FileSystemBackend { replaced = fs.lstatSync(nativeNewPath); } catch {} fs.renameSync(this.safePath(oldPath, false), nativeNewPath); - this.clearDirPathCache(); - if (replaced !== undefined && replaced.nlink <= 1) this.metadata.forget(replaced); + if (replaced !== undefined && replaced.nlink <= 1) + this.metadata.forget(replaced); } link(existingPath: string, newPath: string): void { - fs.linkSync(this.safePath(existingPath), this.safePath(newPath)); + // Resolve intermediate components ourselves, but leave the final source + // component to native link(2). POSIX permits link() either to follow a + // final symlink or to link the symlink inode; native hosts differ here. + fs.linkSync( + this.safePath(existingPath, false), + this.safePath(newPath, false), + ); } symlink(target: string, path: string): void { @@ -425,25 +416,33 @@ export class HostFileSystem implements FileSystemBackend { this.metadata.access(fs.statSync(this.safePath(path)), mode); } - utimensat(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { + utimensat( + path: string, + atimeSec: number, + atimeNsec: number, + mtimeSec: number, + mtimeNsec: number, + ): void { const nativePath = this.safePath(path); if (atimeNsec === UTIME_OMIT && mtimeNsec === UTIME_OMIT) return; const stat = fs.statSync(nativePath); const current = this.metadata.toStatResult(stat); const nowMs = Date.now(); - const atimeMs = atimeNsec === UTIME_OMIT - ? current.atimeMs - : atimeNsec === UTIME_NOW - ? nowMs - : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); - const mtimeMs = mtimeNsec === UTIME_OMIT - ? current.mtimeMs - : mtimeNsec === UTIME_NOW - ? nowMs - : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); + const atimeMs = + atimeNsec === UTIME_OMIT + ? current.atimeMs + : atimeNsec === UTIME_NOW + ? nowMs + : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); + const mtimeMs = + mtimeNsec === UTIME_OMIT + ? current.mtimeMs + : mtimeNsec === UTIME_NOW + ? nowMs + : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); fs.utimesSync(nativePath, atimeMs / 1000, mtimeMs / 1000); - this.metadata.utimens(stat, atimeMs, mtimeMs, fs.statSync(nativePath).ctimeMs); + this.metadata.utimens(stat, atimeMs, mtimeMs, fs.statSync(nativePath)); } // ── Directory iteration ───────────────────────────────────── @@ -462,12 +461,18 @@ export class HostFileSystem implements FileSystemBackend { if (!entry) return null; let dtype = 0; // DT_UNKNOWN - if (entry.isFile()) dtype = 8; // DT_REG - else if (entry.isDirectory()) dtype = 4; // DT_DIR - else if (entry.isSymbolicLink()) dtype = 10; // DT_LNK - else if (entry.isFIFO()) dtype = 1; // DT_FIFO - else if (entry.isSocket()) dtype = 12; // DT_SOCK - else if (entry.isCharacterDevice()) dtype = 2; // DT_CHR + if (entry.isFile()) + dtype = 8; // DT_REG + else if (entry.isDirectory()) + dtype = 4; // DT_DIR + else if (entry.isSymbolicLink()) + dtype = 10; // DT_LNK + else if (entry.isFIFO()) + dtype = 1; // DT_FIFO + else if (entry.isSocket()) + dtype = 12; // DT_SOCK + else if (entry.isCharacterDevice()) + dtype = 2; // DT_CHR else if (entry.isBlockDevice()) dtype = 6; // DT_BLK return { name: entry.name, type: dtype, ino: 0 }; diff --git a/host/src/worker-adapter.ts b/host/src/worker-adapter.ts index 635337f2b2..0f9045a0ef 100644 --- a/host/src/worker-adapter.ts +++ b/host/src/worker-adapter.ts @@ -113,8 +113,10 @@ function currentModuleUrl(): string { return import.meta.url; } -function nodeWorkerStackSizeMb(): number { - const raw = process.env.KANDELO_NODE_WORKER_STACK_SIZE_MB; +/** @internal Exported so the host policy can be validated without spawning a worker. */ +export function nodeWorkerStackSizeMb( + raw = process.env.KANDELO_NODE_WORKER_STACK_SIZE_MB, +): number { if (raw === undefined || raw === "") return DEFAULT_NODE_WORKER_STACK_SIZE_MB; const parsed = Number(raw); if (!Number.isFinite(parsed) || parsed <= 0) { @@ -123,7 +125,11 @@ function nodeWorkerStackSizeMb(): number { return parsed; } -function nodeWorkerOptions(workerData: unknown, options: WorkerOptions = {}): WorkerOptions { +/** @internal Exported so tests can verify resource-limit composition. */ +export function nodeWorkerOptions( + workerData: unknown, + options: WorkerOptions = {}, +): WorkerOptions { return { ...options, workerData, diff --git a/host/test/fixtures/deep-wasm-recursion-worker.mjs b/host/test/fixtures/deep-wasm-recursion-worker.mjs new file mode 100644 index 0000000000..72ce50e580 --- /dev/null +++ b/host/test/fixtures/deep-wasm-recursion-worker.mjs @@ -0,0 +1,7 @@ +import { readFileSync } from "node:fs"; +import { parentPort, workerData } from "node:worker_threads"; + +const bytes = readFileSync(workerData.wasmPath); +const { instance } = await WebAssembly.instantiate(bytes); +const recurse = instance.exports.recurse; +parentPort.postMessage({ result: recurse(workerData.depth) }); diff --git a/host/test/fixtures/deep-wasm-recursion.wat b/host/test/fixtures/deep-wasm-recursion.wat new file mode 100644 index 0000000000..4be3073ea0 --- /dev/null +++ b/host/test/fixtures/deep-wasm-recursion.wat @@ -0,0 +1,14 @@ +(module + (func $recurse (export "recurse") (param $depth i32) (result i32) + local.get $depth + i32.eqz + if (result i32) + i32.const 0 + else + local.get $depth + i32.const 1 + i32.sub + call $recurse + i32.const 1 + i32.add + end)) diff --git a/host/test/global-setup.ts b/host/test/global-setup.ts index b46f1d1c62..bef9ff1ea4 100644 --- a/host/test/global-setup.ts +++ b/host/test/global-setup.ts @@ -42,8 +42,12 @@ const TEST_PROGRAMS = [ "thread-exit-group.c", ]; -/** WAT fixtures used by host/test/wasi-shim.test.ts. */ -const WAT_FIXTURES = ["wasi-args.wat", "wasi-hello.wat"]; +/** WAT fixtures used by host runtime tests. */ +const WAT_FIXTURES = [ + "deep-wasm-recursion.wat", + "wasi-args.wat", + "wasi-hello.wat", +]; function needsRebuild(srcFile: string, outFile: string): boolean { if (!existsSync(outFile)) return true; diff --git a/host/test/vfs/host-fs-path-resolution.test.ts b/host/test/vfs/host-fs-path-resolution.test.ts new file mode 100644 index 0000000000..2da92fc605 --- /dev/null +++ b/host/test/vfs/host-fs-path-resolution.test.ts @@ -0,0 +1,157 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + existsSync, + mkdirSync, + mkdtempSync, + renameSync, + rmSync, + statSync, + symlinkSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { HostFileSystem } from "../../src/vfs/host-fs"; + +const O_WRONLY = 0o1; +const O_CREAT = 0o100; +const O_EXCL = 0o200; +const UTIME_OMIT = 0x3ffffffe; + +describe("HostFileSystem component-wise path resolution", () => { + let top: string; + let root: string; + let outside: string; + let hostFs: HostFileSystem; + + beforeEach(() => { + top = mkdtempSync(join(tmpdir(), "kandelo-host-fs-path-")); + root = join(top, "root"); + outside = join(top, "outside"); + mkdirSync(root); + mkdirSync(outside); + hostFs = new HostFileSystem(root, "/mnt"); + }); + + afterEach(() => { + rmSync(top, { recursive: true, force: true }); + }); + + it("allows ordinary names beginning with two dots and resolves components in order", () => { + writeFileSync(join(root, "..visible"), "visible"); + mkdirSync(join(root, "existing")); + writeFileSync(join(root, "existing", "file"), "data"); + + expect(hostFs.stat("/..visible").size).toBe(7); + expect(() => hostFs.stat("/existing/missing/../file")).toThrow(/ENOENT/); + }); + + it("follows normal relative and in-mount absolute symlinks", () => { + writeFileSync(join(root, "target.txt"), "target"); + symlinkSync("target.txt", join(root, "relative-link")); + symlinkSync("/mnt/target.txt", join(root, "absolute-link")); + + expect(hostFs.stat("/relative-link").size).toBe(6); + expect(hostFs.stat("/absolute-link").size).toBe(6); + expect(hostFs.readlink("/relative-link")).toBe("target.txt"); + + hostFs.unlink("/relative-link"); + expect(existsSync(join(root, "relative-link"))).toBe(false); + expect(existsSync(join(root, "target.txt"))).toBe(true); + }); + + it("does not follow a final symlink for an exclusive create", () => { + symlinkSync("created-through-link", join(root, "exclusive-link")); + + expect(() => + hostFs.open("/exclusive-link", O_WRONLY | O_CREAT | O_EXCL, 0o600), + ).toThrow(/EEXIST/); + expect(existsSync(join(root, "created-through-link"))).toBe(false); + }); + + it("does not follow a dangling final symlink for mkdir", () => { + symlinkSync("created-directory", join(root, "directory-link")); + + expect(() => hostFs.mkdir("/directory-link", 0o755)).toThrow(/EEXIST/); + expect(existsSync(join(root, "created-directory"))).toBe(false); + }); + + it("does not follow a dangling hard-link destination", () => { + writeFileSync(join(root, "source"), "source"); + symlinkSync("created-hard-link", join(root, "destination-link")); + + expect(() => hostFs.link("/source", "/destination-link")).toThrow(/EEXIST/); + expect(existsSync(join(root, "created-hard-link"))).toBe(false); + }); + + it("keeps native link semantics for a symlink source inside the mount", () => { + writeFileSync(join(root, "target"), "target"); + symlinkSync("target", join(root, "source-link")); + + hostFs.link("/source-link", "/linked-symlink"); + + const sourceLink = hostFs.lstat("/source-link"); + const target = hostFs.stat("/target"); + const linked = hostFs.lstat("/linked-symlink"); + expect([sourceLink.ino, target.ino]).toContain(linked.ino); + if ((linked.mode & 0xf000) === 0xa000) { + expect(hostFs.readlink("/linked-symlink")).toBe("target"); + } else { + expect(linked.ino).toBe(target.ino); + } + }); + + it("revalidates an intermediate directory after an external replacement", () => { + mkdirSync(join(root, "cached")); + writeFileSync(join(root, "cached", "inside"), "inside"); + writeFileSync(join(outside, "secret"), "outside-secret"); + + expect(hostFs.stat("/cached/inside").size).toBe(6); + renameSync(join(root, "cached"), join(root, "old-cached")); + symlinkSync(outside, join(root, "cached")); + + expect(() => hostFs.stat("/cached/secret")).toThrow(/EACCES/); + }); +}); + +describe("HostFileSystem utimens metadata", () => { + let root: string; + let hostFs: HostFileSystem; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "kandelo-host-fs-utimens-")); + hostFs = new HostFileSystem(root); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it("preserves an omitted timestamp", () => { + const nativePath = join(root, "timestamps"); + writeFileSync(nativePath, "data"); + utimesSync(nativePath, 10, 20); + const before = statSync(nativePath); + + hostFs.utimensat("/timestamps", 0, UTIME_OMIT, 30, 0); + + const result = hostFs.stat("/timestamps"); + expect(result.atimeMs).toBe(before.atimeMs); + expect(result.mtimeMs).toBe(30_000); + }); + + it("drops timestamp overrides after an external native mutation", () => { + const nativePath = join(root, "externally-mutated"); + writeFileSync(nativePath, "before"); + hostFs.utimensat("/externally-mutated", 1, 0, 2, 0); + expect(hostFs.stat("/externally-mutated").mtimeMs).toBe(2_000); + + writeFileSync(nativePath, "after external mutation"); + const nativeAfter = statSync(nativePath); + + expect(hostFs.stat("/externally-mutated").mtimeMs).toBe( + nativeAfter.mtimeMs, + ); + }); +}); diff --git a/host/test/worker-adapter.test.ts b/host/test/worker-adapter.test.ts index 767a628379..0df11a803e 100644 --- a/host/test/worker-adapter.test.ts +++ b/host/test/worker-adapter.test.ts @@ -1,5 +1,11 @@ +import { fileURLToPath } from "node:url"; import { describe, it, expect } from "vitest"; -import { MockWorkerAdapter } from "../src/worker-adapter"; +import { + MockWorkerAdapter, + NodeWorkerAdapter, + nodeWorkerOptions, + nodeWorkerStackSizeMb, +} from "../src/worker-adapter"; describe("MockWorkerAdapter", () => { it("should create a worker handle and capture workerData", () => { @@ -48,3 +54,75 @@ describe("MockWorkerAdapter", () => { expect(adapter.lastWorker!.sentMessages).toEqual([{ type: "terminate" }]); }); }); + +describe("NodeWorkerAdapter stack policy", () => { + it("uses 32 MiB by default and validates explicit overrides", () => { + expect(nodeWorkerStackSizeMb(undefined)).toBe(32); + expect(nodeWorkerStackSizeMb("48")).toBe(48); + expect(() => nodeWorkerStackSizeMb("0")).toThrow(/invalid/); + expect(() => nodeWorkerStackSizeMb("-1")).toThrow(/invalid/); + expect(() => nodeWorkerStackSizeMb("not-a-number")).toThrow(/invalid/); + }); + + it("preserves other resource limits when setting the stack limit", () => { + expect(nodeWorkerOptions({ pid: 7 }, { + resourceLimits: { maxOldGenerationSizeMb: 64 }, + })).toMatchObject({ + workerData: { pid: 7 }, + resourceLimits: { + maxOldGenerationSizeMb: 64, + stackSizeMb: 32, + }, + }); + }); + + it("runs a deeply recursive Wasm workload inside the configured worker stack", async () => { + const adapter = new NodeWorkerAdapter( + new URL("./fixtures/deep-wasm-recursion-worker.mjs", import.meta.url), + ); + const worker = adapter.createWorker({ + wasmPath: fileURLToPath( + new URL("./fixtures/deep-wasm-recursion.wasm", import.meta.url), + ), + depth: 300_000, + }); + + try { + const result = await new Promise((resolve, reject) => { + worker.on("message", resolve); + worker.on("error", reject); + worker.on("exit", (code) => { + if (code !== 0) reject(new Error(`deep Wasm worker exited ${code}`)); + }); + }); + expect(result).toEqual({ result: 300_000 }); + } finally { + await worker.terminate(); + } + }, 20_000); + + it("supports the default four concurrent process-worker stack reservations", async () => { + const adapter = new NodeWorkerAdapter( + new URL("./fixtures/deep-wasm-recursion-worker.mjs", import.meta.url), + ); + const wasmPath = fileURLToPath( + new URL("./fixtures/deep-wasm-recursion.wasm", import.meta.url), + ); + const workers = Array.from({ length: 4 }, () => + adapter.createWorker({ wasmPath, depth: 100_000 })); + + try { + const results = await Promise.all(workers.map((worker) => + new Promise((resolve, reject) => { + worker.on("message", resolve); + worker.on("error", reject); + worker.on("exit", (code) => { + if (code !== 0) reject(new Error(`deep Wasm worker exited ${code}`)); + }); + }))); + expect(results).toEqual(Array.from({ length: 4 }, () => ({ result: 100_000 }))); + } finally { + await Promise.all(workers.map((worker) => worker.terminate())); + } + }, 20_000); +}); diff --git a/scripts/run-php-upstream-tests.ts b/scripts/run-php-upstream-tests.ts index 0a5b8509f9..ac93bbf357 100644 --- a/scripts/run-php-upstream-tests.ts +++ b/scripts/run-php-upstream-tests.ts @@ -1035,6 +1035,12 @@ class NodePhpRunner implements PhpRunner { { mountPoint: "/php-src", hostPath: this.sourceRoot, + // The source tree is the PHPT workspace. Present it as owned by the + // guest account selected for this run so non-root FPM/CLI coverage + // exercises normal POSIX permission checks instead of a root-owned + // host mount. + uid: this.runUid, + gid: this.runGid, }, { mountPoint: "/kandelo-bin", From 928b087940908246811cd57d871354178f0ca386 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 07:48:10 -0400 Subject: [PATCH 23/29] fix: enforce host runtime contracts across Node and browser Validate exec and spawn modules before committing process state, share process longjmp tags with side modules, keep network and interface behavior host-parallel, validate CPU clock encodings, enforce generation-safe VM deadlines, and route host failures outside guest stderr. --- .../pages/kandelo/kernel-host/live-setup.ts | 9 + crates/kernel/src/syscalls.rs | 98 +++++- docs/posix-status.md | 2 +- examples/clock_getcpuclockid_test.c | 34 ++ host/src/browser-kernel-host.ts | 54 ++- host/src/browser-kernel-protocol.ts | 5 + host/src/browser-kernel-worker-entry.ts | 234 ++++++------- host/src/browser.ts | 1 + host/src/dylink.ts | 77 ++++- host/src/host-diagnostic.ts | 16 + host/src/index.ts | 1 + host/src/kernel-worker.ts | 315 ++++++++++++------ host/src/networking/hostname.ts | 7 +- host/src/networking/tls-network-backend.ts | 37 -- host/src/node-kernel-host.ts | 41 +++ host/src/node-kernel-protocol.ts | 3 + host/src/node-kernel-worker-entry.ts | 241 +++++++------- host/src/vm-interrupt-timer.ts | 230 +++++++++++++ host/src/worker-main.ts | 32 +- host/test/centralized-spawn.test.ts | 32 ++ host/test/centralized-test-helper.ts | 11 +- host/test/clock-id-validation.test.ts | 23 ++ host/test/dylink.test.ts | 101 +++++- host/test/exec-state-tracking.test.ts | 70 +++- host/test/exec.test.ts | 34 ++ host/test/fetch-backend.test.ts | 67 +++- host/test/global-setup.ts | 1 + host/test/host-diagnostic-routing.test.ts | 34 ++ host/test/ifhwaddr.test.ts | 76 +++-- host/test/node-kernel-host-diagnostic.test.ts | 44 +++ host/test/pthread-trap-semantics.test.ts | 6 +- host/test/tcp-backend.test.ts | 40 +++ host/test/time-provider.test.ts | 22 ++ host/test/vm-interrupt-timer.test.ts | 251 ++++++++++++++ host/test/wasm-trap.test.ts | 18 +- .../src/network/wasm32posix/if_indextoname.c | 20 +- .../src/network/wasm32posix/if_nameindex.c | 50 ++- .../src/network/wasm32posix/if_nametoindex.c | 14 +- programs/ifhwaddr.c | 305 ++++++++++++++--- scripts/build-programs.sh | 5 +- 40 files changed, 2128 insertions(+), 533 deletions(-) create mode 100644 examples/clock_getcpuclockid_test.c create mode 100644 host/src/host-diagnostic.ts create mode 100644 host/src/vm-interrupt-timer.ts create mode 100644 host/test/clock-id-validation.test.ts create mode 100644 host/test/host-diagnostic-routing.test.ts create mode 100644 host/test/node-kernel-host-diagnostic.test.ts create mode 100644 host/test/time-provider.test.ts create mode 100644 host/test/vm-interrupt-timer.test.ts diff --git a/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts b/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts index b5f0200291..49905397d4 100644 --- a/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts +++ b/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts @@ -1150,6 +1150,15 @@ async function bootProfile( maxMemoryPages: profile.init?.maxMemoryPages, onStdout: (data) => recordProcessOutput(data, "stdout"), onStderr: (data) => recordProcessOutput(data, "stderr"), + onHostDiagnostic: (diagnostic) => { + if (!isCurrent()) return; + host.pushDmesg({ + t: bootElapsedMs(bootStartedAt), + level: "warn", + facility: "kernel", + msg: diagnostic.message, + }); + }, onProcessEvent: (event) => { if (isCurrent()) host.emitProcessEvent(event); }, onHttpBridgePendingRequests: (count) => { if (isCurrent()) host.setWebPreviewPendingRequests(count); diff --git a/crates/kernel/src/syscalls.rs b/crates/kernel/src/syscalls.rs index f11daff1d7..c7ebc254ce 100644 --- a/crates/kernel/src/syscalls.rs +++ b/crates/kernel/src/syscalls.rs @@ -5547,12 +5547,35 @@ pub fn sys_exit(proc: &mut Process, host: &mut dyn HostIO, status: i32) { } /// Get the current time from the specified clock. +fn is_encoded_process_cpu_clock_id(clock_id: u32) -> bool { + // musl/Linux encode process CPU clocks as (-pid-1)*8 + 2. Real encoded + // IDs are negative clockid_t values; checking only the low three bits + // would incorrectly accept positive IDs such as 10 (produced when an + // invalid negative pid is passed to clock_getcpuclockid()). + (clock_id as i32) < 0 && (clock_id & 7) == 2 +} + +fn host_clock_id(clock_id: u32) -> Result { + use wasm_posix_shared::clock::*; + + match clock_id { + CLOCK_REALTIME | CLOCK_MONOTONIC | CLOCK_PROCESS_CPUTIME_ID + | CLOCK_THREAD_CPUTIME_ID | CLOCK_BOOTTIME => Ok(clock_id), + // clock_getcpuclockid() encodes a process clock as (-pid-1)*8 + 2. + // Kandelo does not yet account CPU time per process, so preserve the + // documented elapsed-time approximation without letting the host + // mistake an encoded ID for CLOCK_REALTIME. + id if is_encoded_process_cpu_clock_id(id) => Ok(CLOCK_PROCESS_CPUTIME_ID), + _ => Err(Errno::EINVAL), + } +} + pub fn sys_clock_gettime( _proc: &Process, host: &mut dyn HostIO, clock_id: u32, ) -> Result { - let (sec, nsec) = host.host_clock_gettime(clock_id)?; + let (sec, nsec) = host.host_clock_gettime(host_clock_id(clock_id)?)?; Ok(WasmTimespec { tv_sec: sec, tv_nsec: nsec, @@ -5581,7 +5604,7 @@ pub fn sys_clock_getres(_proc: &Process, clock_id: u32) -> Result { + id if is_encoded_process_cpu_clock_id(id) => { // Per-process CPU clock: clock_getcpuclockid encodes as (-pid-1)*8 + 2 Ok(WasmTimespec { tv_sec: 0, @@ -5592,8 +5615,8 @@ pub fn sys_clock_getres(_proc: &Process, clock_id: u32) -> Result= 0 and tv_nsec < 1e9. | | `usleep()` | Full | Converts microseconds to sec+nsec, delegates to host_nanosleep. | | `clock_settime()` | Stub | Returns EPERM. Cannot set system clock from Wasm. | diff --git a/examples/clock_getcpuclockid_test.c b/examples/clock_getcpuclockid_test.c new file mode 100644 index 0000000000..a46c1d1539 --- /dev/null +++ b/examples/clock_getcpuclockid_test.c @@ -0,0 +1,34 @@ +#include +#include +#include + +int main(void) { + clockid_t clock_id = 0; + struct timespec ts = {0}; + + /* POSIX returns the error number directly from clock_getcpuclockid(). */ + int rc = clock_getcpuclockid(-2, &clock_id); + if (rc != ESRCH) { + fprintf(stderr, + "FAIL clock_getcpuclockid(-2): rc=%d expected=%d clock=%d\n", + rc, ESRCH, (int)clock_id); + return 1; + } + + errno = 0; + if (clock_gettime((clockid_t)10, &ts) != -1 || errno != EINVAL) { + fprintf(stderr, "FAIL clock_gettime(10): errno=%d expected=%d\n", + errno, EINVAL); + return 1; + } + + errno = 0; + if (clock_getres((clockid_t)10, &ts) != -1 || errno != EINVAL) { + fprintf(stderr, "FAIL clock_getres(10): errno=%d expected=%d\n", + errno, EINVAL); + return 1; + } + + puts("PASS clock id validation"); + return 0; +} diff --git a/host/src/browser-kernel-host.ts b/host/src/browser-kernel-host.ts index 534ad48796..45c752a5b4 100644 --- a/host/src/browser-kernel-host.ts +++ b/host/src/browser-kernel-host.ts @@ -15,6 +15,7 @@ import { import { FramebufferRegistry } from "./framebuffer/registry"; import type { ProcessSnapshot, SyscallTraceEvent } from "./kernel-worker"; import type { + HostDiagnostic, MainToKernelMessage, KernelToMainMessage, } from "./browser-kernel-protocol"; @@ -49,6 +50,8 @@ export interface BrowserKernelOptions { onStdout?: (data: Uint8Array) => void; /** Called when a process writes to stderr */ onStderr?: (data: Uint8Array) => void; + /** Called for host-runtime diagnostics that are not guest stderr. */ + onHostDiagnostic?: (diagnostic: HostDiagnostic) => void; /** Called when a process requests a TCP listener (for service worker bridging) */ onListenTcp?: (pid: number, fd: number, port: number) => void; /** Called when the service-worker HTTP bridge gains or completes preview requests. */ @@ -333,13 +336,26 @@ export class BrowserKernel { this.handleWorkerMessage(e.data as KernelToMainMessage); }; this.kernelWorkerHandle.onerror = (e: ErrorEvent) => { - console.error("[BrowserKernel] Kernel worker error:", e.message); const err = new Error(`Kernel worker error: ${e.message}`); for (const [, { reject }] of this.pendingRequests) { reject(err); } this.pendingRequests.clear(); this.options.onHttpBridgePendingRequests?.(0); + const diagnostic: HostDiagnostic = { + pid: 0, + source: "kernel worker", + message: `[BrowserKernel] kernel worker error: ${e.message}`, + }; + // A worker-level error cannot send a typed message itself. Preserve the + // same callback contract and a visible default without treating the + // failure as guest stderr. + console.error(diagnostic.message); + try { + this.options.onHostDiagnostic?.(diagnostic); + } catch (callbackError) { + console.error("[BrowserKernel] onHostDiagnostic callback failed:", callbackError); + } }; await new Promise((resolve, reject) => { @@ -808,14 +824,18 @@ export class BrowserKernel { /** * Register lazy files: creates stubs in the VFS and forwards metadata - * to the kernel worker so it can materialize on demand via sync XHR. + * to the kernel worker so it can materialize asynchronously before use. */ registerLazyFiles(entries: Array<{ path: string; url: string; size: number; mode?: number }>): void { const fs = this.fs; const lazyEntries: LazyFileEntry[] = []; for (const e of entries) { - const ino = fs.registerLazyFile(e.path, e.url, e.size, e.mode); - lazyEntries.push({ ino, path: e.path, url: e.url, size: e.size }); + fs.registerLazyFile(e.path, e.url, e.size, e.mode); + const entry = fs.getLazyEntry(e.path); + if (!entry) { + throw new Error(`Failed to register lazy VFS file: ${e.path}`); + } + lazyEntries.push(entry); } const requestId = this.nextRequestId++; void this.request(requestId, { @@ -1038,6 +1058,12 @@ export class BrowserKernel { private handleWorkerMessage(msg: KernelToMainMessage): void { switch (msg.type) { + case "ready": + case "init_error": + // The temporary boot listener resolves or rejects initialization. The + // permanent listener also receives these messages, so account for + // them explicitly rather than relying on an implicit fall-through. + break; case "response": { const pending = this.pendingRequests.get(msg.requestId); if (pending) { @@ -1074,6 +1100,15 @@ export class BrowserKernel { case "stderr": this.options.onStderr?.(msg.data); break; + case "host_diagnostic": { + this.options.onHostDiagnostic?.({ + pid: msg.pid, + source: msg.source, + message: msg.message, + ...(msg.status === undefined ? {} : { status: msg.status }), + }); + break; + } case "pty_output": { const cb = this.ptyOutputCallbacks.get(msg.pid); if (cb) { @@ -1120,6 +1155,17 @@ export class BrowserKernel { case "lazy_download": this.emitLazyDownload(msg.event); break; + default: { + // Keep this dispatch coupled to KernelToMainMessage as the protocol + // grows. Runtime values still originate outside TypeScript, so make a + // malformed/unknown worker message visible instead of dropping it. + const exhaustive: never = msg; + void exhaustive; + console.error( + `[BrowserKernel] unknown kernel-worker message type: ${String((msg as { type?: unknown }).type)}`, + ); + break; + } } } diff --git a/host/src/browser-kernel-protocol.ts b/host/src/browser-kernel-protocol.ts index 831aed52f7..5ef1f6c8cd 100644 --- a/host/src/browser-kernel-protocol.ts +++ b/host/src/browser-kernel-protocol.ts @@ -9,8 +9,12 @@ import type { HttpResponse, } from "./networking/in-kernel-http"; import type { LazyDownloadEvent } from "./vfs/memory-fs"; +import type { + HostDiagnosticMessage, +} from "./host-diagnostic"; export type { HttpRequest, HttpResponse }; +export type { HostDiagnostic } from "./host-diagnostic"; // ── Main Thread → Kernel Worker ── @@ -498,6 +502,7 @@ export type KernelToMainMessage = | ExitMessage | StdoutMessage | StderrMessage + | HostDiagnosticMessage | PtyOutputMessage | ListenTcpMessage | FbBindMessage diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index af5b8ac369..292229385c 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -96,6 +96,7 @@ import { removeThreadWorkerRegistryEntry, threadWorkerFailureDisposition, } from "./thread-worker-disposition"; +import { VmInterruptTimerManager } from "./vm-interrupt-timer"; import type { CentralizedWorkerInitMessage, CentralizedThreadInitMessage, @@ -111,6 +112,7 @@ import { type ProcessMemoryLayout, } from "./process-memory"; import type { + HostDiagnostic, MainToKernelMessage, KernelToMainMessage, } from "./browser-kernel-protocol"; @@ -151,14 +153,9 @@ interface ProcessInfo { } const processes = new Map(); const processTeardowns = new Map>(); -interface VmInterruptTimer { - timer?: ReturnType; - process: ProcessInfo; - deadlineMs: number; - timedOutPtr: number; - vmInterruptPtr: number; -} -const vmInterruptTimers = new Map(); +const vmInterruptTimers = new VmInterruptTimerManager( + (pid) => processes.get(pid), +); // Includes standalone thread-worker teardown promises that may outlive the // process map entry they came from. const workerTeardowns = new Set>(); @@ -202,7 +199,18 @@ async function resolveExecutableForLaunch( const shebang = parseShebang(bytes); if (!shebang) { if (!isWasmModuleBytes(bytes)) return { errno: ENOEXEC }; - return { programBytes: bytes, argv }; + let programModule: WebAssembly.Module; + try { + programModule = await WebAssembly.compile(bytes); + } catch (error) { + if (error instanceof WebAssembly.CompileError) return { errno: ENOEXEC }; + throw error; + } + const declaredAbi = extractAbiVersion(bytes); + if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { + return { errno: ENOEXEC }; + } + return { programBytes: bytes, programModule, argv }; } const scriptArgv = [ @@ -228,59 +236,14 @@ const threadWorkers = new Map(); const threadExits = new ThreadExitCoordinator(); const reportedNonzeroProcessExits = new Set(); -function clearVmInterruptTimer(pid: number): void { - const entry = vmInterruptTimers.get(pid); - if (entry?.timer !== undefined) clearTimeout(entry.timer); - vmInterruptTimers.delete(pid); -} - -function scheduleVmInterruptTimer(pid: number, entry: VmInterruptTimer): void { - if (vmInterruptTimers.get(pid) !== entry || processes.get(pid) !== entry.process) return; - const remainingMs = entry.deadlineMs - performance.now(); - if (remainingMs <= 0) { - vmInterruptTimers.delete(pid); - const flags = new Uint8Array(entry.process.memory.buffer); - Atomics.store(flags, entry.timedOutPtr, 1); - Atomics.store(flags, entry.vmInterruptPtr, 1); - return; - } - entry.timer = setTimeout(() => { - entry.timer = undefined; - scheduleVmInterruptTimer(pid, entry); - }, Math.min(0x7fffffff, Math.max(1, remainingMs))); -} - function handleVmInterruptTimer(msg: { pid: number; timedOutPtr: number; vmInterruptPtr: number; seconds: number; -}): void { - clearVmInterruptTimer(msg.pid); - if (!(msg.seconds > 0)) return; - const process = processes.get(msg.pid); - if (!process || !Number.isFinite(msg.seconds)) return; - const flags = new Uint8Array(process.memory.buffer); - if ( - !Number.isSafeInteger(msg.timedOutPtr) || - msg.timedOutPtr < 0 || - msg.timedOutPtr >= flags.length || - !Number.isSafeInteger(msg.vmInterruptPtr) || - msg.vmInterruptPtr < 0 || - msg.vmInterruptPtr >= flags.length - ) return; - // The process worker can be stuck in a CPU-bound Wasm loop, so a timer in - // that worker cannot set cooperative runtime interrupt flags. Run the timer - // from this kernel worker instead; the process memory is shared, matching - // the Node host's VM-interrupt timer path. - const entry: VmInterruptTimer = { - process, - deadlineMs: performance.now() + msg.seconds * 1000, - timedOutPtr: msg.timedOutPtr, - vmInterruptPtr: msg.vmInterruptPtr, - }; - vmInterruptTimers.set(msg.pid, entry); - scheduleVmInterruptTimer(msg.pid, entry); +}, pid: number, process: ProcessInfo): void { + if (msg.pid !== pid) return; + vmInterruptTimers.handleRequest(pid, process, msg); } function delay(ms: number): Promise { @@ -316,7 +279,12 @@ function reportNonzeroProcessExitDiagnostic( `[kernel-worker] nonzero process exit pid=${pid} status=${status} source=${source} argv=${JSON.stringify(info?.argv ?? [])}` + (serviceLog ? `\n${serviceLog}` : "") + `\n${syscalls}`; - console.warn(diagnostic); + reportHostDiagnostic({ + pid, + status, + source, + message: diagnostic, + }, "warn"); } function readServiceLogForProcess(argv: readonly string[] | undefined): string | null { @@ -383,6 +351,15 @@ function post(msg: KernelToMainMessage, transfer?: Transferable[]) { (globalThis as any).postMessage(msg, transfer ?? []); } +function reportHostDiagnostic( + diagnostic: HostDiagnostic, + level: "error" | "warn" = "error", +): void { + if (level === "warn") console.warn(diagnostic.message); + else console.error(diagnostic.message); + post({ type: "host_diagnostic", ...diagnostic }); +} + function reportBridgePendingRequests(): void { post({ type: "http_bridge_pending", count: activeBridgeRequests.size }); } @@ -439,11 +416,10 @@ function respondErrorIfRequested( } function reportWorkerProtocolError(message: string): void { - console.error(`[kernel-worker] ${message}`); - post({ - type: "stderr", + reportHostDiagnostic({ pid: 0, - data: new TextEncoder().encode(`[kernel-worker] ${message}\n`), + source: "worker protocol", + message: `[kernel-worker] ${message}`, }); } @@ -1057,8 +1033,6 @@ function installProcessWorkerListeners( const message = `[kernel-worker] pid=${pid} ${source} -> forcing exit ${status}`; if (status === 0 && source === "worker-main exit message") { console.debug(message); - } else { - console.warn(message); } reportNonzeroProcessExitDiagnostic(pid, status, source); handleExit(pid, status, crashSignum, worker); @@ -1072,9 +1046,15 @@ function installProcessWorkerListeners( // m.status — worker-main posted {type:"exit"}, normal exit path. worker.on("error", (err: Error) => { if (intentionallyTerminated.has(worker as object)) return; - console.error(`[kernel-worker] Worker error pid=${pid}:`, err.message); const signum = classifiedSignalOrFallback(err); - finalize(signalExitStatus(signum), "worker.onerror", signum); + const status = signalExitStatus(signum); + reportHostDiagnostic({ + pid, + status, + source: "worker.onerror", + message: `[kernel-worker] worker error pid=${pid}: ${err.message}`, + }); + finalize(status, "worker.onerror", signum); }); worker.on("exit", (code: number) => { // BrowserWorkerHandle synthesizes an "exit" event when the underlying @@ -1092,27 +1072,23 @@ function installProcessWorkerListeners( }); worker.on("message", (msg: unknown) => { if (intentionallyTerminated.has(worker as object)) return; - if (processes.get(pid)?.worker !== worker) return; + const process = processes.get(pid); + if (!process || process.worker !== worker) return; const m = msg as WorkerToHostMessage; if (m.type === "error") { - console.error(`[kernel-worker] Process error pid=${pid}:`, m.message); - // Forward to host stderr so the demo log shows the actual failure - // ("Kernel worker failed: ..." with the wasm trap or - // instantiation error). Without this, a process death in the - // browser is invisible to the user — only console.error in the - // kernel-worker scope, which most users don't open. Mirrors the - // Node-side handleSpawn message-listener stderr forwarding. - const errBytes = new TextEncoder().encode( - `[process-worker] ${m.message ?? "unknown error"}\n`, - ); - post({ type: "stderr", pid, data: errBytes }); const signum = classifiedSignalOrFallback(m.message); - finalize(classifiedTrapExitStatus(m.message) ?? -1, "worker-main error message", signum); + const status = classifiedTrapExitStatus(m.message) ?? -1; + reportHostDiagnostic({ + pid, + status, + source: "worker-main error message", + message: `[process-worker] ${m.message ?? "unknown error"}`, + }); + finalize(status, "worker-main error message", signum); } else if (m.type === "exit") { finalize(m.status ?? 0, "worker-main exit message"); } else if (m.type === "vm_interrupt_timer") { - if (m.pid !== pid) return; - handleVmInterruptTimer(m); + handleVmInterruptTimer(m, pid, process); } }); } @@ -1214,18 +1190,7 @@ async function handleExec( const resolved = await resolveExecutableForLaunch(path, argv); if (!resolved) return -2; // ENOENT if ("errno" in resolved) return -resolved.errno; - const { programBytes: bytes, argv: launchArgv } = resolved; - let programModule: WebAssembly.Module; - try { - programModule = await WebAssembly.compile(bytes); - } catch (e) { - if (e instanceof WebAssembly.CompileError) return -8; // ENOEXEC - throw e; - } - const declaredAbi = extractAbiVersion(bytes); - if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { - return -8; - } + const { programBytes: bytes, programModule, argv: launchArgv } = resolved; // Preallocate the replacement address space before the irreversible commit. const ptrWidth = detectPtrWidth(bytes); const metadataResult = kernelWorker.validateExecMetadata( @@ -1259,7 +1224,7 @@ async function handleExec( try { const setupResult = kernelWorker.kernelExecSetup(pid, callerTid); if (setupResult < 0) return setupResult; - clearVmInterruptTimer(pid); + vmInterruptTimers.clear(pid); // Invalidate the discarded image synchronously at the commit point. This // keeps stale clone/listener continuations out even if later detach or @@ -1359,10 +1324,11 @@ async function handleExec( const message = err instanceof Error ? err.message : String(err); try { - post({ - type: "stderr", + reportHostDiagnostic({ pid, - data: new TextEncoder().encode(`[exec] post-commit transition failed: ${message}\n`), + status: signalExitStatus(SIGSEGV), + source: "exec post-commit transition", + message: `[exec] post-commit transition failed: ${message}`, }); } catch { // A closed host port must not prevent kernel-side reap. @@ -1381,9 +1347,9 @@ async function handleExec( * Handle SYS_SPAWN (non-forking posix_spawn) on the browser host. * * The kernel has already constructed the child Process descriptor under - * `childPid` with attrs and file actions applied. This callback resolves - * `path` to bytes via the shared MemoryFileSystem, allocates a fresh - * Memory for the child, registers it with the kernel + * `childPid` with attrs and file actions applied. This callback receives the + * preflight's compiled program, allocates a fresh Memory for the child, and + * registers it with the kernel * (`skipKernelCreate: true` — kernel did its half), and spawns a Worker. * * Distinct from handleExec (which replaces the calling worker) and @@ -1398,9 +1364,9 @@ async function handleExec( /** * Pre-flight resolver — see node-kernel-worker-entry.ts:handlePosixSpawnResolve. * Browser-side equivalent: materialize the lazy file (async fetch via - * the memfs lazy-loader, avoiding sync-XHR + SW deadlocks) then read its - * contents from the VFS. Side-effect-free; safe to call on PATH search - * iterations that may not resolve. + * the memfs lazy-loader, avoiding sync-XHR + SW deadlocks), reads its + * contents from the VFS, follows shebangs, and compiles the final Wasm + * module. Safe to call before the kernel applies spawn file actions. */ async function handlePosixSpawnResolve( path: string, @@ -1410,31 +1376,23 @@ async function handlePosixSpawnResolve( } /** - * Launch a worker for a SYS_SPAWN child whose program bytes have already - * been resolved by `handlePosixSpawnResolve`. Mirrors the Node entry's - * `handlePosixSpawn`. + * Launch a worker for a SYS_SPAWN child whose program has already been + * resolved and compiled by `handlePosixSpawnResolve`. Mirrors the Node + * entry's `handlePosixSpawn`. */ async function handlePosixSpawn( childPid: number, - programBytes: ArrayBuffer, - argv: string[], + program: ResolvedSpawnProgram, envp: string[], ): Promise { await waitForProcessTeardowns(); - let programModule: WebAssembly.Module; - try { - programModule = await WebAssembly.compile(programBytes); - } catch (e) { - if (e instanceof WebAssembly.CompileError) return -8; // ENOEXEC - throw e; - } - - // Compilation and unrelated teardown waits yield to the event loop. Keep a - // successfully created zombie, but never resurrect it with a new Worker. + // Unrelated teardown waits yield to the event loop. Keep a successfully + // created zombie, but never resurrect it with a new Worker. if (!kernelWorker.shouldLaunchPendingChild(childPid)) return 0; post({ type: "proc_event", kind: "spawn", pid: childPid }); + const { programBytes, programModule, argv } = program; const ptrWidth = detectPtrWidth(programBytes); const { memory: newMemory, @@ -1535,10 +1493,10 @@ async function handleClone( alloc = processInfo.threadAllocator.allocate(memory); } catch (e) { const message = e instanceof Error ? e.message : String(e); - post({ - type: "stderr", + reportHostDiagnostic({ pid, - data: new TextEncoder().encode(`[kernel-worker] pid=${pid}: ${message}\n`), + source: "clone allocation", + message: `[kernel-worker] pid=${pid}: ${message}`, }); throw e; } @@ -1619,9 +1577,15 @@ async function handleClone( void terminateThreadEntry(); return; } - const text = `[kernel-worker] pid=${pid} tid=${tid}: ${reason}\n`; - post({ type: "stderr", pid, data: new TextEncoder().encode(text) }); const disposition = threadWorkerFailureDisposition(reason); + reportHostDiagnostic({ + pid, + status: disposition.kind === "guest-fatal-trap" + ? disposition.exitStatus + : undefined, + source: "thread worker failure", + message: `[kernel-worker] pid=${pid} tid=${tid}: ${reason}`, + }); kernelWorker.finalizeThreadExit(pid, tid, alloc.channelOffset); void terminateThreadEntry(); if (disposition.kind === "guest-fatal-trap") { @@ -1643,11 +1607,10 @@ async function handleClone( failThread((m as { message?: string }).message ?? "thread error"); } else if (m.type === "vm_interrupt_timer") { if (!isCurrentThreadGeneration() || m.pid !== pid) return; - handleVmInterruptTimer(m); + handleVmInterruptTimer(m, pid, processInfo); } }); threadWorker.on("error", (err: Error) => { - console.error(`[kernel-worker] thread worker error pid=${pid} tid=${tid}:`, err.message); failThread(`worker error: ${err.message ?? err}`); }); @@ -1681,7 +1644,7 @@ async function finishProcessExit( const info = processes.get(pid); if (!info || info.worker !== expectedWorker) return; if (processTeardowns.has(expectedWorker)) return; - clearVmInterruptTimer(pid); + vmInterruptTimers.clear(pid); reportNonzeroProcessExitDiagnostic(pid, exitStatus, "kernel process exit"); const threadedSettleMs = threadedProcessPids.has(pid) @@ -1742,7 +1705,7 @@ async function finishProcessExit( async function handleTerminateProcess(msg: Extract) { const pid = msg.pid; - clearVmInterruptTimer(pid); + vmInterruptTimers.clear(pid); // Terminate thread workers const threads = threadWorkers.get(pid); @@ -1917,10 +1880,7 @@ async function handleDestroy(msg: Extract { kernelWorker.attachKmsStats(msg.crtcId, msg.stats); break; default: { + // Every typed MainToKernelMessage must have a case above. Browser + // tooling also sends a few deliberately out-of-band control messages, + // which remain handled below after the compile-time exhaustiveness + // check. + const exhaustive: never = msg; + void exhaustive; // Handle non-protocol messages (e.g., bridge port transfer) const raw = e.data as any; if (raw?.type === "sysprof_start") { @@ -2254,7 +2220,11 @@ sw.onmessage = (e: MessageEvent) => { }; } } else { - console.warn("[kernel-worker] Unknown message type:", raw?.type); + reportHostDiagnostic({ + pid: 0, + source: "worker protocol", + message: `[kernel-worker] unknown main-thread message type: ${String(raw?.type)}`, + }, "warn"); } } } diff --git a/host/src/browser.ts b/host/src/browser.ts index c6d8597ea6..e294063fd1 100644 --- a/host/src/browser.ts +++ b/host/src/browser.ts @@ -11,6 +11,7 @@ export { centralizedWorkerMain, centralizedThreadWorkerMain, patchWasmForThread export type { MessagePort as WorkerMessagePort } from "./worker-main"; export type { KernelConfig, PlatformIO, StatResult, StatfsResult } from "./types"; export type { WorkerAdapter, WorkerHandle } from "./worker-adapter"; +export type { HostDiagnostic } from "./host-diagnostic"; export type { HostToWorkerMessage, WorkerToHostMessage, WorkerReadyMessage, WorkerExitMessage, WorkerErrorMessage, diff --git a/host/src/dylink.ts b/host/src/dylink.ts index 1bd43deeea..5904ed89e3 100644 --- a/host/src/dylink.ts +++ b/host/src/dylink.ts @@ -324,6 +324,14 @@ export interface LoadSharedLibraryOptions { got: Map; /** Already-loaded libraries for dedup and dependency resolution */ loadedLibraries: Map; + /** + * Process-owned exception tag shared by the main image and every side + * module. When omitted, standalone linker users get one tag lazily and it + * is retained on this options object for subsequent loads. + */ + longjmpTag?: WebAssembly.Tag; + /** Process pointer width, which also determines the __c_longjmp payload. */ + ptrWidth?: 4 | 8; /** Immutable symbol names exported by the main module. */ mainModuleSymbols?: ReadonlySet; /** Present only in a process worker that can drive side-module unwind. */ @@ -336,6 +344,56 @@ export interface LoadSharedLibraryOptions { resolveLibrarySync?: (name: string) => Uint8Array | null; } +type TagConstructor = new ( + descriptor: { parameters: Array<"i32" | "i64"> }, +) => WebAssembly.Tag; + +function tagConstructor(): TagConstructor | undefined { + return (WebAssembly as typeof WebAssembly & { Tag?: TagConstructor }).Tag; +} + +/** Create the exception tag used by one process and all of its side modules. */ +export function createLongjmpTag(ptrWidth: 4 | 8): WebAssembly.Tag | undefined { + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new TypeError(`invalid process pointer width ${String(ptrWidth)}`); + } + const Tag = tagConstructor(); + return Tag + ? new Tag({ parameters: [ptrWidth === 8 ? "i64" : "i32"] }) + : undefined; +} + +/** Reject lookalike values before handing an exception-tag import to Wasm. */ +export function requireLongjmpTag(value: unknown, context: string): WebAssembly.Tag { + const Tag = tagConstructor(); + if (!Tag) { + throw new Error(`${context}: this WebAssembly runtime does not support exception tags`); + } + if (!(value instanceof Tag)) { + throw new TypeError(`${context}: __c_longjmp must be an actual WebAssembly.Tag`); + } + return value; +} + +function validateLongjmpConfiguration(options: LoadSharedLibraryOptions): void { + const ptrWidth = options.ptrWidth ?? 4; + if (ptrWidth !== 4 && ptrWidth !== 8) { + throw new TypeError(`invalid process pointer width ${String(ptrWidth)}`); + } + if (options.longjmpTag !== undefined) { + requireLongjmpTag(options.longjmpTag, "dynamic linker"); + } +} + +function resolveLongjmpTag(options: LoadSharedLibraryOptions): WebAssembly.Tag { + validateLongjmpConfiguration(options); + if (options.longjmpTag !== undefined) return options.longjmpTag; + const ptrWidth = options.ptrWidth ?? 4; + const tag = createLongjmpTag(ptrWidth); + options.longjmpTag = requireLongjmpTag(tag, "dynamic linker"); + return options.longjmpTag; +} + const SIDE_DYNAMIC_LOOKUP_IMPORTS = new Set([ "__wasm_dlopen", "__wasm_dlsym", @@ -447,6 +505,12 @@ function instantiateSharedLibrary( && imp.kind === "function" && SIDE_DYNAMIC_LOOKUP_IMPORTS.has(imp.name) ); + const importsLongjmpTag = moduleImports.some((imp) => + imp.module === "env" + && imp.name === "__c_longjmp" + && (imp.kind as string) === "tag" + ); + const longjmpTag = importsLongjmpTag ? resolveLongjmpTag(options) : undefined; if (presentForkExports.length > 0 && !hasCompleteForkInstrumentation) { const missing = SIDE_MODULE_FORK_EXPORTS.filter((exportName) => @@ -643,16 +707,6 @@ function instantiateSharedLibrary( return entry; }; - // Tag imported by side modules compiled with clang's wasm SjLj lowering - // (`-mllvm -wasm-enable-sjlj`). The host doesn't actually catch these — the - // main process either has its own __c_longjmp tag (LLVM 22) or doesn't use - // SjLj (LLVM 21). A stub Tag lets the side module's import type-check and - // instantiate; behavior at throw time is undefined but the side module - // typically never throws this tag itself. - const longjmpTag = (typeof (WebAssembly as any).Tag === "function") - ? new (WebAssembly as any).Tag({ parameters: ["i32"] }) - : undefined; - let instance: WebAssembly.Instance | null = null; let sideForkState: SideModuleForkState | null = null; const forkState = (): number => { @@ -857,6 +911,7 @@ export async function loadSharedLibrary( wasmBytes: Uint8Array, options: LoadSharedLibraryOptions, ): Promise { + validateLongjmpConfiguration(options); const existing = options.loadedLibraries.get(name); if (existing) return existing; @@ -891,6 +946,7 @@ export function loadSharedLibrarySync( options: LoadSharedLibraryOptions, replay?: DylinkReplayOptions, ): LoadedSharedLibrary { + validateLongjmpConfiguration(options); const existing = options.loadedLibraries.get(name); if (existing) return existing; @@ -938,6 +994,7 @@ export class DynamicLinker { private lastError: string | null = null; constructor(options: LoadSharedLibraryOptions) { + validateLongjmpConfiguration(options); this.options = options; } diff --git a/host/src/host-diagnostic.ts b/host/src/host-diagnostic.ts new file mode 100644 index 0000000000..1b267b39b8 --- /dev/null +++ b/host/src/host-diagnostic.ts @@ -0,0 +1,16 @@ +/** + * Host-runtime diagnostic delivered outside a guest process's fd 2 stream. + * + * `status` is present when the diagnostic describes a process disposition. + * Protocol/setup failures that are not tied to an exit status leave it absent. + */ +export interface HostDiagnostic { + pid: number; + status?: number; + source: string; + message: string; +} + +export interface HostDiagnosticMessage extends HostDiagnostic { + type: "host_diagnostic"; +} diff --git a/host/src/index.ts b/host/src/index.ts index a65bc990b3..c8164c40e8 100644 --- a/host/src/index.ts +++ b/host/src/index.ts @@ -91,6 +91,7 @@ export { WasiShim, WasiExit } from "./wasi-shim"; export { isWasiModule, wasiModuleImportsMemory, wasiModuleDefinesMemory } from "./wasi-detect"; export { NodeKernelHost } from "./node-kernel-host"; export type { NodeKernelHostOptions, SpawnOptions } from "./node-kernel-host"; +export type { HostDiagnostic } from "./host-diagnostic"; export type { MainToKernelMessage, KernelToMainMessage, diff --git a/host/src/kernel-worker.ts b/host/src/kernel-worker.ts index 4d9f0a3887..a24610e601 100644 --- a/host/src/kernel-worker.ts +++ b/host/src/kernel-worker.ts @@ -118,10 +118,12 @@ const E2BIG = 7; const EAGAIN = 11; const EACCES = 13; const EBADF = 9; +const EADDRNOTAVAIL = 99; const EEXIST = 17; const EFAULT = 14; const EIO = 5; const EINVAL = 22; +const ENODEV = 19; const ENOMEM = 12; const ENAMETOOLONG = 36; const ENOENT = 2; @@ -209,9 +211,14 @@ const SIOCGIFCONF = 0x8912; const SIOCGIFHWADDR = 0x8927; const SIOCGIFADDR = 0x8915; const SIOCGIFINDEX = 0x8933; -const VIRTUAL_IFACE_NAME = "eth0"; -const VIRTUAL_IFACE_INDEX = 1; -const ENODEV = 19; +const AF_INET = 2; +const ARPHRD_ETHER = 1; +const ARPHRD_LOOPBACK = 772; +const IF_NAMESIZE = 16; +const VIRTUAL_INTERFACES = [ + { name: "lo", index: 1, loopback: true }, + { name: "eth0", index: 2, loopback: false }, +] as const; /** Ioctl syscall number */ const SYS_IOCTL = ABI_SYSCALLS.Ioctl; @@ -705,6 +712,7 @@ export interface ForkFromThreadContext { export interface ResolvedSpawnProgram { programBytes: ArrayBuffer; + programModule: WebAssembly.Module; argv: string[]; } @@ -712,13 +720,12 @@ export interface SpawnResolveError { errno: number; } -export type SpawnProgramResolution = ArrayBuffer | ResolvedSpawnProgram | SpawnResolveError; +export type SpawnProgramResolution = ResolvedSpawnProgram | SpawnResolveError; function isSpawnResolveError( resolution: SpawnProgramResolution, ): resolution is SpawnResolveError { - return !(resolution instanceof ArrayBuffer) && - "errno" in resolution && + return "errno" in resolution && typeof resolution.errno === "number"; } @@ -767,10 +774,10 @@ export interface CentralizedKernelCallbacks { ) => Promise; /** - * Pre-flight resolution step for SYS_SPAWN. Returns the program bytes - * for `path` (or `{ programBytes, argv }` when resolution rewrites argv, - * e.g. a shebang script), `{ errno }` for a located but unlaunchable - * program, or `null` for ENOENT. **Must NOT have side effects** — + * Pre-flight resolution step for SYS_SPAWN. Returns the validated program + * bytes, their compiled module, and launch argv for `path`, `{ errno }` for + * a located but unlaunchable program, or `null` for ENOENT. **Must NOT have + * side effects** — * `handleSpawn` calls this BEFORE `kernel_spawn_process` so that file * actions never run on a doomed PATH-iteration. POSIX requires * file_actions to run "exactly once," and `posix_spawnp`'s PATH-walk @@ -784,11 +791,12 @@ export interface CentralizedKernelCallbacks { onResolveSpawn?: (path: string, argv: string[]) => Promise; /** - * Launch a worker for the spawned child with already-resolved bytes - * and argv (from `onResolveSpawn`). The kernel has constructed the child Process - * descriptor under `childPid` and applied file actions + attrs by the - * time this is called. The callback instantiates a fresh Worker and - * registers it via `registerProcess({ skipKernelCreate: true })`. + * Launch a worker for the spawned child with the already-resolved bytes, + * compiled module, and argv from `onResolveSpawn`. The kernel has + * constructed the child Process descriptor under `childPid` and applied + * file actions + attrs by the time this is called. The callback instantiates + * a fresh Worker and registers it via + * `registerProcess({ skipKernelCreate: true })`. * * Returns 0 on success, negative errno on failure. On non-zero return * the kernel descriptor is rolled back via `kernel_remove_process`. @@ -797,7 +805,11 @@ export interface CentralizedKernelCallbacks { * `onFork` (which clones the parent's Memory): `onSpawn` always * creates a fresh Memory and runs the new program from `_start`. */ - onSpawn?: (childPid: number, programBytes: ArrayBuffer, argv: string[], envp: string[]) => Promise; + onSpawn?: ( + childPid: number, + program: ResolvedSpawnProgram, + envp: string[], + ) => Promise; /** * Called when a process calls clone (thread creation). The callback should @@ -6070,8 +6082,64 @@ export class CentralizedKernelWorker { } // ---- Network interface ioctl host-side handlers ---- - // The kernel has a single virtual network interface ("eth0") at ifindex 1 - // with a random MAC address generated per kernel instance. + + private finishNetworkIoctl( + channel: ChannelInfo, + retVal = 0, + errno = 0, + ): void { + this.completeChannelRaw(channel, retVal, errno); + this.relistenChannel(channel); + } + + private guestRangeIsValid( + channel: ChannelInfo, + ptr: number, + length: number, + ): boolean { + return Number.isSafeInteger(ptr) && + Number.isSafeInteger(length) && + ptr >= 0 && + length >= 0 && + ptr <= channel.memory.buffer.byteLength - length; + } + + private interfaceAddress( + iface: (typeof VIRTUAL_INTERFACES)[number], + ): Uint8Array | null { + if (iface.loopback) return new Uint8Array([127, 0, 0, 1]); + const address = this.io.network?.localAddress; + return address?.length === 4 ? new Uint8Array(address) : null; + } + + /** + * `struct ifreq` has a 16-byte name followed by a union. The union is 16 + * bytes under wasm32, but its `struct ifmap` member grows to 24 bytes under + * wasm64 because `unsigned long` is pointer-sized. + */ + private ifreqSize(channel: ChannelInfo): number { + return this.getPtrWidth(channel.pid) === 8 ? 40 : 32; + } + + private readIfreqName(channel: ChannelInfo, ifreqPtr: number): string | null { + if (!this.guestRangeIsValid(channel, ifreqPtr, this.ifreqSize(channel))) { + return null; + } + const bytes = new Uint8Array(channel.memory.buffer, ifreqPtr, IF_NAMESIZE); + let end = 0; + while (end < bytes.length && bytes[end] !== 0) end++; + return new TextDecoder().decode(new Uint8Array(bytes.subarray(0, end))); + } + + private writeIfreqName( + processMem: Uint8Array, + ifreqPtr: number, + name: string, + ): void { + const nameBytes = new TextEncoder().encode(name); + processMem.fill(0, ifreqPtr, ifreqPtr + IF_NAMESIZE); + processMem.set(nameBytes.subarray(0, IF_NAMESIZE - 1), ifreqPtr); + } /** * Handle SIOCGIFCONF: enumerate network interfaces. @@ -6080,16 +6148,22 @@ export class CentralizedKernelWorker { * directly — we handle the entire ioctl on the host side. */ private handleIoctlIfconf(channel: ChannelInfo, origArgs: number[]): void { - const processView = new DataView(channel.memory.buffer); - const processMem = new Uint8Array(channel.memory.buffer); const pw = this.getPtrWidth(channel.pid); - - // Read struct ifconf from process memory at arg[2] - // struct ifconf { int ifc_len; union { char *ifc_buf; struct ifreq *ifc_req; }; } - // wasm32: ifc_len at +0 (4B), ifc_buf at +4 (4B) — total 8 bytes - // wasm64: ifc_len at +0 (4B), [4B padding], ifc_buf at +8 (8B) — total 16 bytes const ifconfPtr = origArgs[2]; + const ifconfSize = pw === 8 ? 16 : 8; + if (!this.guestRangeIsValid(channel, ifconfPtr, ifconfSize)) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + + const processView = new DataView(channel.memory.buffer); + const processMem = new Uint8Array(channel.memory.buffer); + const ifreqSize = this.ifreqSize(channel); const ifcLen = processView.getInt32(ifconfPtr, true); + if (ifcLen < 0) { + this.finishNetworkIoctl(channel, -EINVAL, EINVAL); + return; + } let ifcBuf: number; if (pw === 8) { ifcBuf = Number(processView.getBigUint64(ifconfPtr + 8, true)); @@ -6097,32 +6171,41 @@ export class CentralizedKernelWorker { ifcBuf = processView.getUint32(ifconfPtr + 4, true); } - // sizeof(struct ifreq) = 32 (16 name + 16 sockaddr union, no pointers — same on both) - const SIZEOF_IFREQ = 32; - - if (ifcLen >= SIZEOF_IFREQ && ifcBuf !== 0) { - // Write one ifreq entry for the virtual interface into process memory at ifc_buf - const nameBytes = new TextEncoder().encode(VIRTUAL_IFACE_NAME); - processMem.set(nameBytes, ifcBuf); - processMem.fill(0, ifcBuf + nameBytes.length, ifcBuf + 16); // pad ifr_name - - // ifr_addr: AF_INET (2) with 127.0.0.1 — just needs to be a valid sockaddr - processMem.fill(0, ifcBuf + 16, ifcBuf + SIZEOF_IFREQ); - processView.setUint16(ifcBuf + 16, 2, true); // AF_INET - processMem[ifcBuf + 20] = 127; // sin_addr = 127.0.0.1 - processMem[ifcBuf + 21] = 0; - processMem[ifcBuf + 22] = 0; - processMem[ifcBuf + 23] = 1; + if (ifcBuf === 0) { + processView.setInt32( + ifconfPtr, + VIRTUAL_INTERFACES.length * ifreqSize, + true, + ); + this.finishNetworkIoctl(channel); + return; + } - // Update ifc_len to actual bytes written - processView.setInt32(ifconfPtr, SIZEOF_IFREQ, true); - } else { - // No space or null buffer + if (ifcLen < ifreqSize) { processView.setInt32(ifconfPtr, 0, true); + this.finishNetworkIoctl(channel); + return; } - this.completeChannelRaw(channel, 0, 0); - this.relistenChannel(channel); + const capacity = Math.floor(ifcLen / ifreqSize); + const count = Math.min(capacity, VIRTUAL_INTERFACES.length); + const bytesToWrite = count * ifreqSize; + if (!this.guestRangeIsValid(channel, ifcBuf, bytesToWrite)) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + + for (let i = 0; i < count; i++) { + const iface = VIRTUAL_INTERFACES[i]; + const entryPtr = ifcBuf + i * ifreqSize; + this.writeIfreqName(processMem, entryPtr, iface.name); + processMem.fill(0, entryPtr + IF_NAMESIZE, entryPtr + ifreqSize); + processView.setUint16(entryPtr + IF_NAMESIZE, AF_INET, true); + const address = this.interfaceAddress(iface); + if (address) processMem.set(address, entryPtr + IF_NAMESIZE + 4); + } + processView.setInt32(ifconfPtr, bytesToWrite, true); + this.finishNetworkIoctl(channel); } /** @@ -6130,23 +6213,23 @@ export class CentralizedKernelWorker { * struct ifreq at arg[2]: ifr_name[16] + union; ifr_ifindex lives at +16. */ private handleIoctlIfname(channel: ChannelInfo, origArgs: number[]): void { + const ifreqPtr = origArgs[2]; + if (!this.guestRangeIsValid(channel, ifreqPtr, this.ifreqSize(channel))) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } const processView = new DataView(channel.memory.buffer); const processMem = new Uint8Array(channel.memory.buffer); - const ifreqPtr = origArgs[2]; const ifindex = processView.getInt32(ifreqPtr + 16, true); + const iface = VIRTUAL_INTERFACES.find((candidate) => candidate.index === ifindex); - if (ifindex !== VIRTUAL_IFACE_INDEX) { - this.completeChannelRaw(channel, -ENODEV, ENODEV); - this.relistenChannel(channel); + if (!iface) { + this.finishNetworkIoctl(channel, -ENODEV, ENODEV); return; } - const nameBytes = new TextEncoder().encode(VIRTUAL_IFACE_NAME); - processMem.set(nameBytes, ifreqPtr); - processMem.fill(0, ifreqPtr + nameBytes.length, ifreqPtr + 16); - - this.completeChannelRaw(channel, 0, 0); - this.relistenChannel(channel); + this.writeIfreqName(processMem, ifreqPtr, iface.name); + this.finishNetworkIoctl(channel); } /** @@ -6155,41 +6238,71 @@ export class CentralizedKernelWorker { * Returns the virtual MAC in ifr_hwaddr.sa_data[0..5]. */ private handleIoctlIfhwaddr(channel: ChannelInfo, origArgs: number[]): void { + const ifreqPtr = origArgs[2]; + const name = this.readIfreqName(channel, ifreqPtr); + if (name === null) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + const iface = VIRTUAL_INTERFACES.find((candidate) => candidate.name === name); + if (!iface) { + this.finishNetworkIoctl(channel, -ENODEV, ENODEV); + return; + } const processView = new DataView(channel.memory.buffer); const processMem = new Uint8Array(channel.memory.buffer); - const ifreqPtr = origArgs[2]; - // Write ifr_hwaddr at offset 16 from ifreq start: - // sa_family = ARPHRD_ETHER (1) - // sa_data[0..5] = MAC address - processMem.fill(0, ifreqPtr + 16, ifreqPtr + 32); // clear sockaddr - processView.setUint16(ifreqPtr + 16, 1, true); // ARPHRD_ETHER - processMem.set(this.virtualMacAddress, ifreqPtr + 18); // MAC address + processMem.fill( + 0, + ifreqPtr + IF_NAMESIZE, + ifreqPtr + this.ifreqSize(channel), + ); + processView.setUint16( + ifreqPtr + IF_NAMESIZE, + iface.loopback ? ARPHRD_LOOPBACK : ARPHRD_ETHER, + true, + ); + if (!iface.loopback) { + processMem.set(this.virtualMacAddress, ifreqPtr + IF_NAMESIZE + 2); + } - this.completeChannelRaw(channel, 0, 0); - this.relistenChannel(channel); + this.finishNetworkIoctl(channel); } /** * Handle SIOCGIFADDR: get interface address. * struct ifreq at arg[2]: ifr_name[16] + ifr_addr (struct sockaddr, 16 bytes) - * Returns 127.0.0.1 for the virtual interface. + * Returns the selected virtual interface's assigned IPv4 address. */ private handleIoctlIfaddr(channel: ChannelInfo, origArgs: number[]): void { + const ifreqPtr = origArgs[2]; + const name = this.readIfreqName(channel, ifreqPtr); + if (name === null) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); + return; + } + const iface = VIRTUAL_INTERFACES.find((candidate) => candidate.name === name); + if (!iface) { + this.finishNetworkIoctl(channel, -ENODEV, ENODEV); + return; + } + const address = this.interfaceAddress(iface); + if (!address) { + this.finishNetworkIoctl(channel, -EADDRNOTAVAIL, EADDRNOTAVAIL); + return; + } const processView = new DataView(channel.memory.buffer); const processMem = new Uint8Array(channel.memory.buffer); - const ifreqPtr = origArgs[2]; - // Write ifr_addr at offset 16: AF_INET + 127.0.0.1 - processMem.fill(0, ifreqPtr + 16, ifreqPtr + 32); - processView.setUint16(ifreqPtr + 16, 2, true); // AF_INET - processMem[ifreqPtr + 20] = 127; // sin_addr = 127.0.0.1 - processMem[ifreqPtr + 21] = 0; - processMem[ifreqPtr + 22] = 0; - processMem[ifreqPtr + 23] = 1; + processMem.fill( + 0, + ifreqPtr + IF_NAMESIZE, + ifreqPtr + this.ifreqSize(channel), + ); + processView.setUint16(ifreqPtr + IF_NAMESIZE, AF_INET, true); + processMem.set(address, ifreqPtr + IF_NAMESIZE + 4); - this.completeChannelRaw(channel, 0, 0); - this.relistenChannel(channel); + this.finishNetworkIoctl(channel); } /** @@ -6197,26 +6310,25 @@ export class CentralizedKernelWorker { * struct ifreq at arg[2]: ifr_name[16] + union; ifr_ifindex lives at +16. */ private handleIoctlIfindex(channel: ChannelInfo, origArgs: number[]): void { - const processView = new DataView(channel.memory.buffer); - const processMem = new Uint8Array(channel.memory.buffer); const ifreqPtr = origArgs[2]; - const nul = processMem.indexOf(0, ifreqPtr); - const end = nul >= ifreqPtr && nul < ifreqPtr + 16 ? nul : ifreqPtr + 16; - // Browser TextDecoder rejects SharedArrayBuffer-backed views. Copy the - // guest ifr_name bytes before decoding; ioctl handlers must work for - // process memories backed by SAB. - const name = new TextDecoder().decode(new Uint8Array(processMem.subarray(ifreqPtr, end))); - - if (name !== VIRTUAL_IFACE_NAME) { - this.completeChannelRaw(channel, -ENODEV, ENODEV); - this.relistenChannel(channel); + const name = this.readIfreqName(channel, ifreqPtr); + if (name === null) { + this.finishNetworkIoctl(channel, -EFAULT, EFAULT); return; } + const iface = VIRTUAL_INTERFACES.find((candidate) => candidate.name === name); - processView.setInt32(ifreqPtr + 16, VIRTUAL_IFACE_INDEX, true); + if (!iface) { + this.finishNetworkIoctl(channel, -ENODEV, ENODEV); + return; + } - this.completeChannelRaw(channel, 0, 0); - this.relistenChannel(channel); + new DataView(channel.memory.buffer).setInt32( + ifreqPtr + IF_NAMESIZE, + iface.index, + true, + ); + this.finishNetworkIoctl(channel); } private handleWritev(channel: ChannelInfo, syscallNr: number, origArgs: number[]): void { @@ -7324,14 +7436,14 @@ export class CentralizedKernelWorker { return; } - // ── PRE-FLIGHT: resolve program bytes BEFORE calling the kernel ── + // ── PRE-FLIGHT: resolve and compile BEFORE calling the kernel ── // POSIX requires file_actions to run "exactly once." `posix_spawnp`'s // PATH search emits one `posix_spawn` per candidate; if we let the // kernel apply file_actions on each iteration, the side effects // (e.g. `addopen(O_EXCL)`) accumulate and the second iteration sees // its own state from the first. Resolve bytes via the host's // side-effect-free preflight first; only call the kernel if the - // program actually exists. + // program actually exists and compiles. const resolveSpawnProgram = async (): Promise => { const resolved = await this.callbacks.onResolveSpawn!(path, argv); if (resolved || rawPath === path || !rawPath || rawPath.startsWith("/")) { @@ -7356,10 +7468,8 @@ export class CentralizedKernelWorker { this.completeChannel(channel, SYS_SPAWN, origArgs, undefined, -1, resolved.errno >>> 0); return; } - const programBytes = resolved instanceof ArrayBuffer ? resolved : resolved.programBytes; - const launchArgv = resolved instanceof ArrayBuffer ? argv : resolved.argv; this.handleSpawnAfterResolve( - channel, origArgs, parentPid, pidOutPtr, blobBytes, blobLen, launchArgv, envp, programBytes, + channel, origArgs, parentPid, pidOutPtr, blobBytes, blobLen, resolved, envp, ); }).catch((err) => { if (!this.isAsyncChannelProcessActive(channel)) return; @@ -7370,8 +7480,8 @@ export class CentralizedKernelWorker { /** * Continuation of `handleSpawn` after `onResolveSpawn` has returned - * actual program bytes. Now safe to ask the kernel to build the child - * (which will apply file_actions exactly once). + * validated, compiled program. Now safe to ask the kernel to build the + * child (which will apply file_actions exactly once). */ private handleSpawnAfterResolve( channel: ChannelInfo, @@ -7380,9 +7490,8 @@ export class CentralizedKernelWorker { pidOutPtr: number, blobBytes: Uint8Array, blobLen: number, - argv: string[], + program: ResolvedSpawnProgram, envp: string[], - programBytes: ArrayBuffer, ): void { // ── Copy blob to kernel scratch ── const kernelMem = new Uint8Array(this.kernelMemory!.buffer); @@ -7431,14 +7540,14 @@ export class CentralizedKernelWorker { try { this.inheritHostFdMirrors(parentPid, childPid, false); launch = Promise.resolve( - this.callbacks.onSpawn!(childPid, programBytes, argv, envp), + this.callbacks.onSpawn!(childPid, program, envp), ); } catch (err) { rollbackSpawn(5, err); return; } - // ── Launch the worker async (with already-resolved bytes) ── + // ── Launch the worker async (with the precompiled program) ── launch.then((rc) => { if (rc < 0) { rollbackSpawn((-rc) >>> 0); diff --git a/host/src/networking/hostname.ts b/host/src/networking/hostname.ts index 9cd79ecdf1..bb46dcf04d 100644 --- a/host/src/networking/hostname.ts +++ b/host/src/networking/hostname.ts @@ -93,12 +93,7 @@ export function validateSyntheticDnsHostname( ) { return; } - if (lowerName === "localhost") return; - if ( - !lowerName.includes(".") || - lowerName === "invalid" || - lowerName.endsWith(".invalid") - ) { + if (lowerName === "invalid" || lowerName.endsWith(".invalid")) { throw nameNotFoundError(hostname); } } diff --git a/host/src/networking/tls-network-backend.ts b/host/src/networking/tls-network-backend.ts index 18a0405720..fe48172346 100644 --- a/host/src/networking/tls-network-backend.ts +++ b/host/src/networking/tls-network-backend.ts @@ -146,17 +146,6 @@ function formatHttpResponse( return result; } -function headersFromRawHeaderString(rawHeaders: string): Headers { - const headers = new Headers(); - for (const line of rawHeaders.split(/\r?\n/)) { - if (!line) continue; - const colon = line.indexOf(":"); - if (colon <= 0) continue; - headers.append(line.slice(0, colon).trim(), line.slice(colon + 1).trim()); - } - return headers; -} - function corsProxyFetchUrl(corsProxyUrl: string, targetUrl: string): string { if (targetUrl.startsWith(corsProxyUrl)) { return targetUrl; @@ -594,32 +583,6 @@ export class TlsNetworkBackend implements NetworkIO { const fetchBody: Uint8Array | undefined = body && body.length > 0 ? new Uint8Array(body) as Uint8Array : undefined; - const isWindowContext = typeof document !== "undefined"; - if (typeof XMLHttpRequest !== "undefined" && !isWindowContext) { - try { - const xhr = new XMLHttpRequest(); - xhr.open(method, url, false); - xhr.responseType = "arraybuffer"; - for (const [key, value] of fetchHeaders) { - xhr.setRequestHeader(key, value); - } - xhr.send(fetchBody); - conn.responseBuf = formatHttpResponse( - xhr.status, - xhr.statusText, - headersFromRawHeaderString(xhr.getAllResponseHeaders()), - xhr.response ?? new ArrayBuffer(0), - ); - conn.fetchDone = true; - conn.sendBuf = new Uint8Array(0); - return data.length; - } catch { - // Fall through to fetch when synchronous XHR is unavailable or rejected - // by the current browser context. Dedicated workers use the blocking - // path to match POSIX socket reads; other contexts retain async fetch. - } - } - const doFetch = async () => { try { const response = await fetch(url, { diff --git a/host/src/node-kernel-host.ts b/host/src/node-kernel-host.ts index 2f2ac00220..c97fc1a87c 100644 --- a/host/src/node-kernel-host.ts +++ b/host/src/node-kernel-host.ts @@ -20,6 +20,7 @@ import { createRequire } from "node:module"; import { Worker as NodeThreadWorker } from "node:worker_threads"; import { resolveBinary } from "./binary-resolver"; import type { + HostDiagnostic, MainToKernelMessage, KernelToMainMessage, ResolveExecRequestMessage, @@ -61,6 +62,8 @@ export interface NodeKernelHostOptions { onStdout?: (pid: number, data: Uint8Array) => void; /** Called when a process writes to stderr */ onStderr?: (pid: number, data: Uint8Array) => void; + /** Called for host-runtime diagnostics that are not guest stderr. */ + onHostDiagnostic?: (diagnostic: HostDiagnostic) => void; /** Called when a process writes PTY output */ onPtyOutput?: (pid: number, data: Uint8Array) => void; /** Called when a process is spawned, execs a new program, or exits. @@ -152,6 +155,20 @@ export class NodeKernelHost { reject(error); } this.pendingRequests.clear(); + const diagnostic: HostDiagnostic = { + pid: 0, + source: "kernel worker", + message: `[NodeKernelHost] kernel worker error: ${error.message}`, + }; + // A worker-level error cannot send a typed message itself. Preserve the + // same callback contract and a visible default without treating the + // failure as guest stderr. + console.error(diagnostic.message); + try { + this.options.onHostDiagnostic?.(diagnostic); + } catch (callbackError) { + console.error("[NodeKernelHost] onHostDiagnostic callback failed:", callbackError); + } }); // Send init and wait for ready @@ -488,6 +505,10 @@ export class NodeKernelHost { private handleWorkerMessage(msg: KernelToMainMessage): void { switch (msg.type) { + case "ready": + // The temporary init listener resolves readiness. The permanent + // listener also receives the message, so account for it explicitly. + break; case "response": { const pending = this.pendingRequests.get(msg.requestId); if (pending) { @@ -532,12 +553,32 @@ export class NodeKernelHost { case "stderr": this.options.onStderr?.(msg.pid, msg.data); break; + case "host_diagnostic": { + this.options.onHostDiagnostic?.({ + pid: msg.pid, + source: msg.source, + message: msg.message, + ...(msg.status === undefined ? {} : { status: msg.status }), + }); + break; + } case "pty_output": this.options.onPtyOutput?.(msg.pid, msg.data); break; case "resolve_exec": this.handleResolveExec(msg); break; + default: { + // Keep this dispatch coupled to KernelToMainMessage as the protocol + // grows. Runtime values still originate outside TypeScript, so make a + // malformed/unknown worker message visible instead of dropping it. + const exhaustive: never = msg; + void exhaustive; + console.error( + `[NodeKernelHost] unknown kernel-worker message type: ${String((msg as { type?: unknown }).type)}`, + ); + break; + } } } diff --git a/host/src/node-kernel-protocol.ts b/host/src/node-kernel-protocol.ts index 3ce55aace5..c1e5145e19 100644 --- a/host/src/node-kernel-protocol.ts +++ b/host/src/node-kernel-protocol.ts @@ -11,8 +11,10 @@ * See docs/plans/2026-04-30-external-kernel-http-request-interface.md. */ import type { HttpRequest, HttpResponse } from "./networking/in-kernel-http"; +import type { HostDiagnosticMessage } from "./host-diagnostic"; export type { HttpRequest, HttpResponse }; +export type { HostDiagnostic } from "./host-diagnostic"; // ── Main Thread → Kernel Worker ── @@ -266,6 +268,7 @@ export type KernelToMainMessage = | ExitMessage | StdoutMessage | StderrMessage + | HostDiagnosticMessage | PtyOutputMessage | ResolveExecRequestMessage | ProcEventMessage; diff --git a/host/src/node-kernel-worker-entry.ts b/host/src/node-kernel-worker-entry.ts index 660546919e..fb26895119 100644 --- a/host/src/node-kernel-worker-entry.ts +++ b/host/src/node-kernel-worker-entry.ts @@ -10,8 +10,8 @@ * Main → Worker: init, spawn, append_stdin_data, set_stdin_data, * pty_write, pty_resize, terminate_process, destroy, * resolve_exec_response - * Worker → Main: ready, response, exit, stdout, stderr, pty_output, - * resolve_exec + * Worker → Main: ready, response, exit, stdout, stderr, host_diagnostic, + * pty_output, resolve_exec */ import { parentPort } from "node:worker_threads"; import { readFileSync, existsSync, mkdtempSync, rmSync } from "node:fs"; @@ -59,6 +59,7 @@ import { removeThreadWorkerRegistryEntry, threadWorkerFailureDisposition, } from "./thread-worker-disposition"; +import { VmInterruptTimerManager } from "./vm-interrupt-timer"; import { computeProcessMemoryLayout, createProcessMemory, @@ -73,6 +74,7 @@ import type { WorkerToHostMessage, } from "./worker-protocol"; import type { + HostDiagnostic, MainToKernelMessage, KernelToMainMessage, InitMessage, @@ -116,15 +118,11 @@ interface ProcessInfo { } const processes = new Map(); const processTeardowns = new Map>(); -interface VmInterruptTimer { - timer?: ReturnType; - process: ProcessInfo; - deadlineMs: number; - timedOutPtr: number; - vmInterruptPtr: number; -} -const vmInterruptTimers = new Map(); +const vmInterruptTimers = new VmInterruptTimerManager( + (pid) => processes.get(pid), +); const reportedExits = new Set(); +const reportedNonzeroProcessExits = new Set(); // Workers terminated by the kernel-worker entry itself (handleExit / // handleExec / handleTerminate). The crash safety-net listener checks @@ -150,11 +148,16 @@ function installCrashSafetyNet( if (intentionallyTerminated.has(worker as object)) return; const cur = processes.get(pid); if (!cur || cur.worker !== worker) return; // already torn down or replaced - const errBytes = new TextEncoder().encode( - `[process-worker] pid=${pid} crashed (worker exit code=${code}, no SYS_exit_group from wasm)\n`, - ); - post({ type: "stderr", pid, data: errBytes }); - void finalizeProcessWorker(pid, worker, signalExitStatus(SIGSEGV), SIGSEGV); + const status = signalExitStatus(SIGSEGV); + reportHostDiagnostic({ + pid, + status, + source: "worker exit event", + message: + `[process-worker] pid=${pid} crashed ` + + `(worker exit code=${code}, no SYS_exit_group from wasm)`, + }); + void finalizeProcessWorker(pid, worker, status, SIGSEGV); }); } @@ -201,29 +204,16 @@ async function terminateThreadWorkers(pid: number): Promise { function reportProcessExit(pid: number, status: number): void { if (reportedExits.has(pid)) return; reportedExits.add(pid); - post({ type: "exit", pid, status }); -} - -function clearVmInterruptTimer(pid: number): void { - const entry = vmInterruptTimers.get(pid); - if (entry?.timer !== undefined) clearTimeout(entry.timer); - vmInterruptTimers.delete(pid); -} - -function scheduleVmInterruptTimer(pid: number, entry: VmInterruptTimer): void { - if (vmInterruptTimers.get(pid) !== entry || processes.get(pid) !== entry.process) return; - const remainingMs = entry.deadlineMs - performance.now(); - if (remainingMs <= 0) { - vmInterruptTimers.delete(pid); - const flags = new Uint8Array(entry.process.memory.buffer); - Atomics.store(flags, entry.timedOutPtr, 1); - Atomics.store(flags, entry.vmInterruptPtr, 1); - return; + if (status !== 0 && !reportedNonzeroProcessExits.has(pid)) { + reportedNonzeroProcessExits.add(pid); + reportHostDiagnostic({ + pid, + status, + source: "process exit", + message: `[node-kernel-worker] nonzero process exit pid=${pid} status=${status}`, + }, "warn"); } - entry.timer = setTimeout(() => { - entry.timer = undefined; - scheduleVmInterruptTimer(pid, entry); - }, Math.min(0x7fffffff, Math.max(1, remainingMs))); + post({ type: "exit", pid, status }); } function handleVmInterruptTimer(msg: { @@ -231,28 +221,9 @@ function handleVmInterruptTimer(msg: { timedOutPtr: number; vmInterruptPtr: number; seconds: number; -}, process: ProcessInfo): void { - if (processes.get(msg.pid) !== process) return; - clearVmInterruptTimer(msg.pid); - if (!(msg.seconds > 0)) return; - if (!Number.isFinite(msg.seconds)) return; - const flags = new Uint8Array(process.memory.buffer); - if ( - !Number.isSafeInteger(msg.timedOutPtr) || - msg.timedOutPtr < 0 || - msg.timedOutPtr >= flags.length || - !Number.isSafeInteger(msg.vmInterruptPtr) || - msg.vmInterruptPtr < 0 || - msg.vmInterruptPtr >= flags.length - ) return; - const entry: VmInterruptTimer = { - process, - deadlineMs: performance.now() + msg.seconds * 1000, - timedOutPtr: msg.timedOutPtr, - vmInterruptPtr: msg.vmInterruptPtr, - }; - vmInterruptTimers.set(msg.pid, entry); - scheduleVmInterruptTimer(msg.pid, entry); +}, pid: number, process: ProcessInfo): void { + if (msg.pid !== pid) return; + vmInterruptTimers.handleRequest(pid, process, msg); } function signalFromExitStatus(exitStatus: number): number | null { @@ -292,7 +263,7 @@ async function finalizeProcessWorker( if (intentionallyTerminated.has(worker as object)) return; const cur = processes.get(pid); if (!cur || cur.worker !== worker) return; - clearVmInterruptTimer(pid); + vmInterruptTimers.clear(pid); // Synthesize a signal-style reap *before* `deactivateProcess` in // case the worker died without sending SYS_EXIT_GROUP (uncaught @@ -340,9 +311,13 @@ function finalizeProcessWorkerError( ): void { if (intentionallyTerminated.has(worker as object)) return; if (processes.get(pid)?.worker !== worker) return; - const errBytes = new TextEncoder().encode(`[process-worker] ${message ?? "unknown error"}\n`); - post({ type: "stderr", pid, data: errBytes }); const { exitStatus, signum } = processWorkerErrorDisposition(message); + reportHostDiagnostic({ + pid, + status: exitStatus, + source: "worker-main error message", + message: `[process-worker] ${message ?? "unknown error"}`, + }); void finalizeProcessWorker(pid, worker, exitStatus, signum); } @@ -355,9 +330,13 @@ function finalizeUnexpectedWorkerError( if (intentionallyTerminated.has(worker as object)) return; if (processes.get(pid)?.worker !== worker) return; const message = err instanceof Error ? (err.message ?? String(err)) : String(err); - const errBytes = new TextEncoder().encode(`[kernel-worker] pid=${pid}: ${label}: ${message}\n`); - post({ type: "stderr", pid, data: errBytes }); const { exitStatus, signum } = unexpectedWorkerCrashDisposition(err); + reportHostDiagnostic({ + pid, + status: exitStatus, + source: label, + message: `[kernel-worker] pid=${pid}: ${label}: ${message}`, + }); void finalizeProcessWorker(pid, worker, exitStatus, signum); } @@ -365,6 +344,23 @@ function post(msg: KernelToMainMessage) { port.postMessage(msg); } +function reportHostDiagnostic( + diagnostic: HostDiagnostic, + level: "error" | "warn" = "error", +): void { + if (level === "warn") console.warn(diagnostic.message); + else console.error(diagnostic.message); + post({ type: "host_diagnostic", ...diagnostic }); +} + +function reportWorkerProtocolError(message: string): void { + reportHostDiagnostic({ + pid: 0, + source: "worker protocol", + message: `[node-kernel-worker] ${message}`, + }); +} + function respond(requestId: number, result: unknown) { post({ type: "response", requestId, result }); } @@ -561,7 +557,18 @@ async function resolveExecutableForLaunch( const shebang = parseShebang(bytes); if (!shebang) { if (!isWasmModuleBytes(bytes)) return { errno: ENOEXEC }; - return { programBytes: bytes, argv }; + let programModule: WebAssembly.Module; + try { + programModule = await WebAssembly.compile(bytes); + } catch (error) { + if (error instanceof WebAssembly.CompileError) return { errno: ENOEXEC }; + throw error; + } + const declaredAbi = extractAbiVersion(bytes); + if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { + return { errno: ENOEXEC }; + } + return { programBytes: bytes, programModule, argv }; } const scriptArgv = [ @@ -841,7 +848,7 @@ function handleSpawn(msg: SpawnMessage) { void finalizeProcessWorker(pid, worker, m.status ?? 0); } else if (m.type === "vm_interrupt_timer" && m.pid === pid) { const process = processes.get(pid); - if (process?.worker === worker) handleVmInterruptTimer(m, process); + if (process?.worker === worker) handleVmInterruptTimer(m, pid, process); } }); @@ -937,7 +944,9 @@ async function handleFork( void finalizeProcessWorker(childPid, childWorker, m.status ?? 0); } else if (m.type === "vm_interrupt_timer" && m.pid === childPid) { const process = processes.get(childPid); - if (process?.worker === childWorker) handleVmInterruptTimer(m, process); + if (process?.worker === childWorker) { + handleVmInterruptTimer(m, childPid, process); + } } }); @@ -960,18 +969,7 @@ async function handleExec( const resolved = await resolveExecutableForLaunch(path, argv); if (!resolved) return -2; // ENOENT if ("errno" in resolved) return -resolved.errno; - const { programBytes, argv: launchArgv } = resolved; - let programModule: WebAssembly.Module; - try { - programModule = await getCompiledProgramModule(programBytes); - } catch (e) { - if (e instanceof WebAssembly.CompileError) return -8; // ENOEXEC - throw e; - } - const declaredAbi = extractAbiVersion(programBytes); - if (declaredAbi !== null && declaredAbi !== kernelWorker.getKernelAbiVersion()) { - return -8; // ENOEXEC: known ABI mismatch is a truthful launch failure - } + const { programBytes, programModule, argv: launchArgv } = resolved; const newPtrWidth = detectPtrWidth(programBytes); const metadataResult = kernelWorker.validateExecMetadata( launchArgv, @@ -1000,7 +998,7 @@ async function handleExec( try { const setupResult = kernelWorker.kernelExecSetup(pid, callerTid); if (setupResult < 0) return setupResult; - clearVmInterruptTimer(pid); + vmInterruptTimers.clear(pid); // From this point onward the old image cannot resume. Invalidate its // channels and async continuations immediately, before any other @@ -1084,7 +1082,9 @@ async function handleExec( void finalizeProcessWorker(pid, replacementWorker!, m.status ?? 0); } else if (m.type === "vm_interrupt_timer" && m.pid === pid) { const process = processes.get(pid); - if (process?.worker === replacementWorker) handleVmInterruptTimer(m, process); + if (process && process.worker === replacementWorker) { + handleVmInterruptTimer(m, pid, process); + } } }); @@ -1109,10 +1109,11 @@ async function handleExec( const message = err instanceof Error ? err.message : String(err); try { - post({ - type: "stderr", + reportHostDiagnostic({ pid, - data: new TextEncoder().encode(`[exec] post-commit transition failed: ${message}\n`), + status: signalExitStatus(SIGSEGV), + source: "exec post-commit transition", + message: `[exec] post-commit transition failed: ${message}`, }); } catch { // A closed host port must not prevent kernel-side reap. @@ -1146,7 +1147,9 @@ async function handleExec( /** * Pre-flight resolver for SYS_SPAWN. Side-effect-free: looks up program * bytes for `path` (via the same execPrograms map + main-thread fallback - * `resolveExec` already uses for execve). Returns null on ENOENT. + * `resolveExec` already uses for execve), follows shebangs, and compiles the + * final Wasm module. Returns null on ENOENT and `{ errno }` when the located + * target cannot be launched. * * `handleSpawn` in `host/src/kernel-worker.ts` calls this BEFORE * `kernel_spawn_process` so that file_actions (which the kernel runs @@ -1161,31 +1164,23 @@ async function handlePosixSpawnResolve( } /** - * Launch a worker for a SYS_SPAWN child whose program bytes have already - * been resolved by `handlePosixSpawnResolve`. The kernel has built the - * child Process descriptor + applied file actions by the time we get - * here, so this just allocates a Memory, registers the process, and - * spawns the worker. + * Launch a worker for a SYS_SPAWN child whose program has already been + * resolved and compiled by `handlePosixSpawnResolve`. The kernel has built + * the child Process descriptor + applied file actions by the time we get + * here, so this just allocates a Memory, registers the process, and spawns + * the worker. */ async function handlePosixSpawn( childPid: number, - programBytes: ArrayBuffer, - argv: string[], + program: ResolvedSpawnProgram, envp: string[], ): Promise { - let programModule: WebAssembly.Module; - try { - programModule = await getCompiledProgramModule(programBytes); - } catch (e) { - if (e instanceof WebAssembly.CompileError) return -8; // ENOEXEC - throw e; - } - - // Compilation yields. Preserve a child that became a zombie while waiting, - // but do not resurrect it by registering a new execution generation. + // Preserve a child that became a zombie before launch, but do not resurrect + // it by registering a new execution generation. if (!kernelWorker.shouldLaunchPendingChild(childPid)) return 0; post({ type: "proc_event", kind: "spawn", pid: childPid }); + const { programBytes, programModule, argv } = program; const ptrWidth = detectPtrWidth(programBytes); const { memory, @@ -1240,7 +1235,9 @@ async function handlePosixSpawn( void finalizeProcessWorker(childPid, newWorker, m.status ?? 0); } else if (m.type === "vm_interrupt_timer" && m.pid === childPid) { const process = processes.get(childPid); - if (process?.worker === newWorker) handleVmInterruptTimer(m, process); + if (process?.worker === newWorker) { + handleVmInterruptTimer(m, childPid, process); + } } }); @@ -1290,10 +1287,10 @@ async function handleClone( alloc = processInfo.threadAllocator.allocate(memory); } catch (e) { const message = e instanceof Error ? e.message : String(e); - post({ - type: "stderr", + reportHostDiagnostic({ pid, - data: new TextEncoder().encode(`[kernel-worker] pid=${pid}: ${message}\n`), + source: "clone allocation", + message: `[kernel-worker] pid=${pid}: ${message}`, }); throw e; } @@ -1370,9 +1367,15 @@ async function handleClone( void terminateThreadEntry(); return; } - const text = `[kernel-worker] pid=${pid} tid=${tid}: ${reason}\n`; - post({ type: "stderr", pid, data: new TextEncoder().encode(text) }); const disposition = threadWorkerFailureDisposition(reason); + reportHostDiagnostic({ + pid, + status: disposition.kind === "guest-fatal-trap" + ? disposition.exitStatus + : undefined, + source: "thread worker failure", + message: `[kernel-worker] pid=${pid} tid=${tid}: ${reason}`, + }); kernelWorker.finalizeThreadExit(pid, tid, alloc.channelOffset); void terminateThreadEntry(); if (disposition.kind === "guest-fatal-trap") { @@ -1392,7 +1395,7 @@ async function handleClone( failThread(m.message); } else if (m.type === "vm_interrupt_timer") { if (isCurrentThreadGeneration() && m.pid === pid) { - handleVmInterruptTimer(m, processInfo); + handleVmInterruptTimer(m, pid, processInfo); } } }); @@ -1417,7 +1420,7 @@ async function finishProcessExit( if (!expectedWorker) return; const info = processes.get(pid); if (!info || info.worker !== expectedWorker) return; - clearVmInterruptTimer(pid); + vmInterruptTimers.clear(pid); const existingTeardown = processTeardowns.get(expectedWorker); if (existingTeardown) { @@ -1465,7 +1468,7 @@ async function finishProcessExit( async function handleTerminate(msg: TerminateProcessMessage) { const pid = msg.pid; - clearVmInterruptTimer(pid); + vmInterruptTimers.clear(pid); // Terminate thread workers const threads = threadWorkers.get(pid); @@ -1502,7 +1505,7 @@ async function handleTerminate(msg: TerminateProcessMessage) { async function handleDestroy(msg: { requestId: number }) { const processEntries = [...processes.entries()]; for (const [pid, info] of processEntries) { - clearVmInterruptTimer(pid); + vmInterruptTimers.clear(pid); await terminateThreadWorkers(pid); await terminateTrackedWorker(info.worker); try { kernelWorker.unregisterProcess(pid); } catch {} @@ -1518,12 +1521,10 @@ async function handleDestroy(msg: { requestId: number }) { } } processes.clear(); - for (const entry of vmInterruptTimers.values()) { - if (entry.timer !== undefined) clearTimeout(entry.timer); - } - vmInterruptTimers.clear(); + vmInterruptTimers.clearAll(); processTeardowns.clear(); reportedExits.clear(); + reportedNonzeroProcessExits.clear(); threadModuleCache.clear(); threadWorkers.clear(); ptyByPid.clear(); @@ -1668,5 +1669,13 @@ port.on("message", (msg: MainToKernelMessage) => { case "kms_attach_stats": kernelWorker.attachKmsStats(msg.crtcId, msg.stats); break; + default: { + const exhaustive: never = msg; + void exhaustive; + reportWorkerProtocolError( + `unknown main-thread message type: ${String((msg as { type?: unknown }).type)}`, + ); + break; + } } }); diff --git a/host/src/vm-interrupt-timer.ts b/host/src/vm-interrupt-timer.ts new file mode 100644 index 0000000000..9886097156 --- /dev/null +++ b/host/src/vm-interrupt-timer.ts @@ -0,0 +1,230 @@ +/** + * A process-scoped cooperative VM-interrupt timer. + * + * Process workers cannot reliably run their own timer while executing a + * CPU-bound Wasm loop. The kernel worker owns this timer instead and writes + * the runtime's interrupt flags through the process's shared memory. + * + * The generation object is deliberately part of every entry. A numeric PID + * can be reused by exec or a later process, so neither a queued timer callback + * nor a stale worker message may act on whatever generation happens to own the + * PID later. + */ + +export const MAX_VM_INTERRUPT_TIMER_DELAY_MS = 0x7fffffff; + +export interface VmInterruptProcessGeneration { + readonly memory: WebAssembly.Memory; +} + +export interface VmInterruptTimerRequest { + timedOutPtr: number; + vmInterruptPtr: number; + seconds: number; +} + +export interface VmInterruptTimerScheduler> { + /** Monotonic milliseconds. */ + now(): number; + set(callback: () => void, delayMs: number): Handle; + clear(handle: Handle): void; +} + +interface TimerEntry { + generation: Generation; + deadlineMs: number; + timedOutPtr: number; + vmInterruptPtr: number; + handle?: Handle; +} + +function defaultScheduler(): VmInterruptTimerScheduler { + return { + now: () => performance.now(), + set: (callback, delayMs) => setTimeout(callback, delayMs), + clear: (handle) => clearTimeout(handle), + }; +} + +function sharedFlags( + generation: VmInterruptProcessGeneration, + timedOutPtr: number, + vmInterruptPtr: number, +): Uint8Array | null { + // TypeScript's WebAssembly.Memory.buffer declaration is ArrayBuffer even + // when the memory descriptor used `shared: true`; narrow from unknown so a + // real SharedArrayBuffer remains representable without an impossible + // ArrayBuffer & SharedArrayBuffer intersection. + const buffer: unknown = generation.memory.buffer; + if ( + typeof SharedArrayBuffer === "undefined" || + !(buffer instanceof SharedArrayBuffer) + ) { + return null; + } + if ( + !Number.isSafeInteger(timedOutPtr) || + timedOutPtr < 0 || + timedOutPtr >= buffer.byteLength || + !Number.isSafeInteger(vmInterruptPtr) || + vmInterruptPtr < 0 || + vmInterruptPtr >= buffer.byteLength + ) { + return null; + } + return new Uint8Array(buffer); +} + +export class VmInterruptTimerManager< + Generation extends VmInterruptProcessGeneration, + Handle = ReturnType, +> { + private readonly entries = new Map>(); + + constructor( + private readonly currentGeneration: (pid: number) => Generation | undefined, + private readonly scheduler: VmInterruptTimerScheduler = + defaultScheduler() as VmInterruptTimerScheduler, + ) {} + + /** + * Apply the runtime hook's arm/cancel request for the listener-owned PID. + * Non-positive durations cancel the current generation's timer. + */ + handleRequest( + pid: number, + generation: Generation, + request: VmInterruptTimerRequest, + ): boolean { + if (request.seconds > 0) { + return this.arm(pid, generation, request); + } + return this.cancel(pid, generation); + } + + /** Replace any existing timer for the same current process generation. */ + arm( + pid: number, + generation: Generation, + request: VmInterruptTimerRequest, + ): boolean { + if (this.currentGeneration(pid) !== generation) return false; + + // A new request replaces the previous timer even when the new request is + // malformed. This matches timer-set semantics and prevents an old deadline + // from surviving a rejected re-arm. + this.clear(pid); + + if (!Number.isFinite(request.seconds) || !(request.seconds > 0)) return false; + if (!sharedFlags(generation, request.timedOutPtr, request.vmInterruptPtr)) { + return false; + } + + const now = this.scheduler.now(); + const delayMs = request.seconds * 1000; + const deadlineMs = now + delayMs; + if (!Number.isFinite(now) || !Number.isFinite(delayMs) || !Number.isFinite(deadlineMs)) { + return false; + } + + const entry: TimerEntry = { + generation, + deadlineMs, + timedOutPtr: request.timedOutPtr, + vmInterruptPtr: request.vmInterruptPtr, + }; + this.entries.set(pid, entry); + this.schedule(pid, entry); + return true; + } + + /** Cancel only when the caller still owns the current PID generation. */ + cancel(pid: number, generation: Generation): boolean { + if (this.currentGeneration(pid) !== generation) return false; + this.clear(pid, generation); + return true; + } + + /** Clear a PID timer, optionally restricted to one exact generation. */ + clear(pid: number, generation?: Generation): boolean { + const entry = this.entries.get(pid); + if (!entry || (generation !== undefined && entry.generation !== generation)) { + return false; + } + if (entry.handle !== undefined) { + this.scheduler.clear(entry.handle); + entry.handle = undefined; + } + this.entries.delete(pid); + return true; + } + + clearAll(): void { + for (const [pid] of this.entries) this.clear(pid); + } + + get activeCount(): number { + return this.entries.size; + } + + private schedule(pid: number, entry: TimerEntry): void { + if ( + this.entries.get(pid) !== entry || + this.currentGeneration(pid) !== entry.generation + ) { + this.discardIfCurrent(pid, entry); + return; + } + + const remainingMs = entry.deadlineMs - this.scheduler.now(); + if (remainingMs <= 0) { + this.fire(pid, entry); + return; + } + + // Browser and Node timers clamp/overflow beyond a signed 32-bit delay. + // Schedule in chunks and recompute against the monotonic deadline after + // every wake. Ceil prevents a fractional delay from firing early. + const delayMs = Math.min( + MAX_VM_INTERRUPT_TIMER_DELAY_MS, + Math.max(1, Math.ceil(remainingMs)), + ); + const handle = this.scheduler.set(() => { + if (this.entries.get(pid) !== entry || entry.handle !== handle) return; + entry.handle = undefined; + this.schedule(pid, entry); + }, delayMs); + entry.handle = handle; + } + + private fire(pid: number, entry: TimerEntry): void { + if ( + this.entries.get(pid) !== entry || + this.currentGeneration(pid) !== entry.generation + ) { + this.discardIfCurrent(pid, entry); + return; + } + + const flags = sharedFlags( + entry.generation, + entry.timedOutPtr, + entry.vmInterruptPtr, + ); + this.entries.delete(pid); + entry.handle = undefined; + if (!flags) return; + + Atomics.store(flags, entry.timedOutPtr, 1); + Atomics.store(flags, entry.vmInterruptPtr, 1); + } + + private discardIfCurrent(pid: number, entry: TimerEntry): void { + if (this.entries.get(pid) !== entry) return; + if (entry.handle !== undefined) { + this.scheduler.clear(entry.handle); + entry.handle = undefined; + } + this.entries.delete(pid); + } +} diff --git a/host/src/worker-main.ts b/host/src/worker-main.ts index 3a8f6c584d..f9b40f73cc 100644 --- a/host/src/worker-main.ts +++ b/host/src/worker-main.ts @@ -11,10 +11,12 @@ import type { WorkerToHostMessage, } from "./worker-protocol"; import { + createLongjmpTag, DynamicLinker, FORK_CAP_DYLINK_MAIN, forkInstrumentRoleAvailable, readForkInstrumentCapabilityClaim, + requireLongjmpTag, type LoadedSharedLibrary, type SideModuleForkState, } from "./dylink"; @@ -213,6 +215,7 @@ function buildDlopenImports( getStackPointer: () => WebAssembly.Global | undefined, getInstance: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8, + longjmpTag: WebAssembly.Tag | undefined, mainHasDylinkForkRole: boolean, ): DlopenSupport { let linker: DynamicLinker | null = null; @@ -312,7 +315,7 @@ function buildDlopenImports( // not be shadowed by main exports. const RESERVED = new Set([ "memory", "__indirect_function_table", - "__memory_base", "__table_base", "__stack_pointer", + "__memory_base", "__table_base", "__stack_pointer", "__c_longjmp", ]); const globalSymbols = new Map(); const inst = getInstance(); @@ -380,6 +383,8 @@ function buildDlopenImports( globalSymbols, got: new Map(), loadedLibraries, + longjmpTag, + ptrWidth, mainModuleSymbols, sideModuleFork, sideModuleForkUnavailableReason: !mainHasDylinkForkRole @@ -668,6 +673,7 @@ function buildImportObject( dlopenImports?: Record, getInstance?: () => WebAssembly.Instance | undefined, ptrWidth: 4 | 8 = 4, + longjmpTag?: WebAssembly.Tag, postVmInterruptTimer?: ( timedOutPtr: number, vmInterruptPtr: number, @@ -692,14 +698,14 @@ function buildImportObject( } } - // llvm/lld ≥22 emit __c_longjmp as a tag import for setjmp users; instantiation fails silently without it. + // LLVM/lld >= 22 import this tag for setjmp users. The process owns its + // identity so a longjmp thrown through a side module can be caught by the + // main image (and vice versa). if (moduleImports.some(i => i.module === "env" && i.name === "__c_longjmp" && (i.kind as string) === "tag")) { - const Tag = (WebAssembly as typeof WebAssembly & { - Tag?: new (descriptor: { parameters: string[] }) => WebAssembly.Tag; - }).Tag; - if (Tag) { - envImports.__c_longjmp = new Tag({ parameters: ["i32"] }) as unknown as WebAssembly.ExportValue; - } + envImports.__c_longjmp = requireLongjmpTag( + longjmpTag, + "process module", + ) as unknown as WebAssembly.ExportValue; } // Add dlopen imports if provided @@ -1109,6 +1115,7 @@ export async function centralizedWorkerMain( } // --- SDK module path (existing) --- + const processLongjmpTag = createLongjmpTag(ptrWidth); let kernelExitStatus: number | null = null; const kernelImports = buildKernelImports( memory, @@ -1164,10 +1171,11 @@ export async function centralizedWorkerMain( () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, + processLongjmpTag, hasDylinkForkRole, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth, + () => processInstance ?? undefined, ptrWidth, processLongjmpTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -1328,10 +1336,11 @@ export async function centralizedWorkerMain( () => processInstance?.exports.__stack_pointer as WebAssembly.Global | undefined, () => processInstance ?? undefined, ptrWidth, + processLongjmpTag, false, ); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, dlopenSupport.imports, - () => processInstance ?? undefined, ptrWidth, + () => processInstance ?? undefined, ptrWidth, processLongjmpTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", @@ -2035,8 +2044,9 @@ export async function centralizedThreadWorkerMain( ); }; } + const threadLongjmpTag = createLongjmpTag(ptrWidth); const importObject = buildImportObject(module, memory, kernelImports, channelOffset, undefined, - () => threadInstance, ptrWidth, + () => threadInstance, ptrWidth, threadLongjmpTag, (timedOutPtr, vmInterruptPtr, seconds) => { port.postMessage({ type: "vm_interrupt_timer", diff --git a/host/test/centralized-spawn.test.ts b/host/test/centralized-spawn.test.ts index ee078c5b92..7566b0ce99 100644 --- a/host/test/centralized-spawn.test.ts +++ b/host/test/centralized-spawn.test.ts @@ -100,4 +100,36 @@ describe("non-forking posix_spawn", () => { rmSync(tempDir, { recursive: true, force: true }); } }); + + it("reports ENOEXEC for malformed Wasm before creating the spawn child", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "kandelo-malformed-wasm-")); + const malformedWasm = join(tempDir, "malformed.wasm"); + try { + // Valid magic/version, followed by a truncated type section. A magic-only + // check accepts this; WebAssembly compilation must reject it in spawn's + // side-effect-free preflight, before the kernel applies file actions. + writeFileSync(malformedWasm, Buffer.from([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + 0x01, 0x01, 0xff, + ])); + + const result = await runCentralizedProgram({ + programPath: spawnSmokeWasm, + argv: ["spawn-smoke", "/usr/bin/malformed.wasm"], + execPrograms: new Map([ + ["/usr/bin/malformed.wasm", malformedWasm], + ]), + useDefaultRootfs: false, + timeout: 30_000, + }); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain("Exec format error"); + expect(result.stderr).not.toContain("Centralized worker failed"); + expect(result.stderr).not.toContain("WebAssembly.compile()"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); }); diff --git a/host/test/centralized-test-helper.ts b/host/test/centralized-test-helper.ts index 1241e68b9c..cba55b7a1d 100644 --- a/host/test/centralized-test-helper.ts +++ b/host/test/centralized-test-helper.ts @@ -21,6 +21,7 @@ import { type ProcessMemoryLayout, } from "../src/process-memory"; import { NodeKernelHost } from "../src/node-kernel-host"; +import type { HostDiagnostic } from "../src/host-diagnostic"; import type { CentralizedWorkerInitMessage, CentralizedThreadInitMessage, WorkerToHostMessage } from "../src/worker-protocol"; import type { PlatformIO } from "../src/types"; @@ -145,6 +146,8 @@ export interface RunProgramResult { exitCode: number; stdout: string; stderr: string; + /** Host-owned lifecycle/protocol diagnostics, never guest fd 2 bytes. */ + hostDiagnostics: HostDiagnostic[]; /** Raw stdout bytes (for binary output like compressed data) */ stdoutBytes: Uint8Array; /** Per-process fork counter for the spawned process, captured immediately @@ -179,6 +182,7 @@ async function runInWorkerThread(options: RunProgramOptions): Promise { stderr += new TextDecoder().decode(data); }, + onHostDiagnostic: (diagnostic) => { + hostDiagnostics.push(diagnostic); + }, }); await host.init(); @@ -270,7 +277,7 @@ async function runInWorkerThread(options: RunProgramOptions): Promise { + it("maps an invalid negative pid to ESRCH and rejects positive clock ID 10", async () => { + const result = await runCentralizedProgram({ + programPath: program, + argv: ["clock_getcpuclockid_test"], + useDefaultRootfs: false, + timeout: 10_000, + }); + + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stdout).toContain("PASS clock id validation"); + expect(result.stderr).toBe(""); + }); +}); diff --git a/host/test/dylink.test.ts b/host/test/dylink.test.ts index 32ca004382..b493423e3b 100644 --- a/host/test/dylink.test.ts +++ b/host/test/dylink.test.ts @@ -4,6 +4,7 @@ import { describe, it, expect } from "vitest"; import { + createLongjmpTag, parseDylinkSection, loadSharedLibrary, loadSharedLibrarySync, @@ -70,13 +71,14 @@ function buildDylinkWat( forkCapabilities?: number, tableSize = 0, memorySize = 0, + wat2wasmFlags: string[] = [], ): Uint8Array { const dir = join(tmpdir(), "wasm-dylink-wat-test"); mkdirSync(dir, { recursive: true }); const watPath = join(dir, `${name}.wat`); const wasmPath = join(dir, `${name}.wasm`); writeFileSync(watPath, wat); - execFileSync("wat2wasm", ["--enable-threads", watPath, "-o", wasmPath], { + execFileSync("wat2wasm", ["--enable-threads", ...wat2wasmFlags, watPath, "-o", wasmPath], { stdio: "pipe", }); const module = new Uint8Array(readFileSync(wasmPath)); @@ -103,6 +105,103 @@ function buildDylinkWat( ); } +describe.skipIf(typeof WebAssembly.Tag !== "function")("longjmp tag identity", () => { + const cases = [ + { ptrWidth: 4 as const, wasmType: "i32", value: 37 }, + { ptrWidth: 8 as const, wasmType: "i64", value: 37n }, + ]; + + it.each(cases)( + "shares one process-owned $wasmType tag with a side module", + ({ ptrWidth, wasmType, value }) => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $longjmp (import "env" "__c_longjmp") (param ${wasmType})) + (func (export "throw_longjmp") (param $value ${wasmType}) + local.get $value + throw $longjmp)) + `, `longjmp-${wasmType}`, undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + const longjmpTag = createLongjmpTag(ptrWidth)!; + options.ptrWidth = ptrWidth; + options.longjmpTag = longjmpTag; + // A same-named main export is not authoritative for this reserved tag. + options.globalSymbols.set("__c_longjmp", () => 0); + + const lib = loadSharedLibrarySync(`liblongjmp-${wasmType}.so`, wasmBytes, options); + const throwLongjmp = lib.exports.throw_longjmp as (arg: number | bigint) => void; + let caught: unknown; + try { + throwLongjmp(value); + } catch (error) { + caught = error; + } + + const WasmException = (WebAssembly as typeof WebAssembly & { + Exception: new (...args: unknown[]) => Error; + }).Exception; + expect(caught).toBeInstanceOf(WasmException); + const exception = caught as Error & { + is: (tag: WebAssembly.Tag) => boolean; + getArg: (tag: WebAssembly.Tag, index: number) => unknown; + }; + expect(exception.is(longjmpTag)).toBe(true); + expect(exception.getArg(longjmpTag, 0)).toBe(value); + }, + ); + + it.each(cases)( + "creates one pointer-width-aware $wasmType fallback for standalone linkers", + ({ ptrWidth, wasmType, value }) => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $longjmp (import "env" "__c_longjmp") (param ${wasmType})) + (func (export "throw_longjmp") (param $value ${wasmType}) + local.get $value + throw $longjmp)) + `, `fallback-longjmp-${wasmType}`, undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + options.ptrWidth = ptrWidth; + + const first = loadSharedLibrarySync(`libfallback-${wasmType}-one.so`, wasmBytes, options); + const fallbackTag = options.longjmpTag!; + expect(fallbackTag).toBeInstanceOf(WebAssembly.Tag); + const second = loadSharedLibrarySync(`libfallback-${wasmType}-two.so`, wasmBytes, options); + expect(options.longjmpTag).toBe(fallbackTag); + + for (const lib of [first, second]) { + let caught: unknown; + try { + (lib.exports.throw_longjmp as (arg: number | bigint) => void)(value); + } catch (error) { + caught = error; + } + const exception = caught as { + is: (tag: WebAssembly.Tag) => boolean; + getArg: (tag: WebAssembly.Tag, index: number) => unknown; + }; + expect(exception.is(fallbackTag)).toBe(true); + expect(exception.getArg(fallbackTag, 0)).toBe(value); + } + }, + ); + + it("rejects a lookalike tag before side-module instantiation", () => { + const wasmBytes = buildDylinkWat(` + (module + (import "env" "memory" (memory 1 100 shared)) + (tag $longjmp (import "env" "__c_longjmp") (param i32))) + `, "invalid-longjmp-tag", undefined, 0, 0, ["--enable-exceptions"]); + const options = createSideForkLoadOptions(); + options.longjmpTag = {} as WebAssembly.Tag; + + expect(() => loadSharedLibrarySync("libinvalid-longjmp.so", wasmBytes, options)) + .toThrow(/__c_longjmp must be an actual WebAssembly\.Tag/); + }); +}); + describe.skipIf(!hasCompiler())("dylink.0 parser", () => { it("parses a simple shared library", () => { const wasmBytes = buildSharedLib( diff --git a/host/test/exec-state-tracking.test.ts b/host/test/exec-state-tracking.test.ts index 55c08205b7..a8ed46812d 100644 --- a/host/test/exec-state-tracking.test.ts +++ b/host/test/exec-state-tracking.test.ts @@ -9,6 +9,7 @@ import { CH_ARGS, CH_DATA_SIZE, CH_RETURN, + HOST_INTERCEPTED_SYSCALLS, } from "../src/generated/abi"; describe("exec host-state transition", () => { @@ -249,8 +250,8 @@ describe("exec host-state transition", () => { bytes.set(path, pathPtr); const blobPtr = 0x200; bytes.fill(0, blobPtr, blobPtr + 40); - let resolveProgram!: (value: ArrayBuffer) => void; - const program = new Promise((resolve) => { + let resolveProgram!: (value: ReturnType) => void; + const program = new Promise>((resolve) => { resolveProgram = resolve; }); const kernelSpawn = vi.fn(() => 100); @@ -270,7 +271,7 @@ describe("exec host-state transition", () => { worker.handleSpawn(channel, [pathPtr, path.length, blobPtr, 40, 0, 0]); worker.processes.get(7).channels = []; - resolveProgram(new ArrayBuffer(8)); + resolveProgram(resolvedProgram()); await Promise.resolve(); await Promise.resolve(); @@ -279,6 +280,46 @@ describe("exec host-state transition", () => { expect(completeChannel).not.toHaveBeenCalled(); }); + it("rejects an unlaunchable spawn before creating a child or applying file actions", async () => { + const memory = new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }); + const channel = createChannel(7, memory, 0); + const bytes = new Uint8Array(memory.buffer); + const pathPtr = 0x100; + const path = new TextEncoder().encode("/bin/malformed"); + bytes.set(path, pathPtr); + const blobPtr = 0x200; + bytes.fill(0, blobPtr, blobPtr + 40); + const kernelSpawn = vi.fn(() => 100); + const onSpawn = vi.fn(async () => 0); + const completeChannel = vi.fn(); + const worker = createWorker({ + processes: new Map([[7, { channels: [channel], memory }]]), + callbacks: { + onResolveSpawn: vi.fn(async () => ({ errno: 8 })), + onSpawn, + }, + completeChannel, + kernelInstance: { + exports: { kernel_spawn_process: kernelSpawn }, + }, + }); + + worker.handleSpawn(channel, [pathPtr, path.length, blobPtr, 40, 0, 0]); + await Promise.resolve(); + await Promise.resolve(); + + expect(kernelSpawn).not.toHaveBeenCalled(); + expect(onSpawn).not.toHaveBeenCalled(); + expect(completeChannel).toHaveBeenCalledWith( + channel, + HOST_INTERCEPTED_SYSCALLS.SYS_SPAWN, + [pathPtr, path.length, blobPtr, 40, 0, 0], + undefined, + -1, + 8, + ); + }); + it("keeps a created spawn child but suppresses stale parent completion", async () => { const memory = new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }); const channel = createChannel(7, memory, 0); @@ -289,9 +330,11 @@ describe("exec host-state transition", () => { const kernelSpawn = vi.fn(() => 100); const removeProcess = vi.fn(); const completeChannel = vi.fn(); + const onSpawn = vi.fn(() => spawned); + const program = resolvedProgram(); const worker = createWorker({ processes: new Map([[7, { channels: [channel], memory }]]), - callbacks: { onSpawn: vi.fn(() => spawned) }, + callbacks: { onSpawn }, completeChannel, kernelMemory: new WebAssembly.Memory({ initial: 1 }), scratchOffset: 0, @@ -311,10 +354,10 @@ describe("exec host-state transition", () => { 0, new Uint8Array(40), 40, + program, [], - [], - new ArrayBuffer(8), ); + expect(onSpawn).toHaveBeenCalledWith(100, program, []); worker.processes.get(7).channels = []; finishSpawn(0); await Promise.resolve(); @@ -370,9 +413,8 @@ describe("exec host-state transition", () => { 0, new Uint8Array(40), 40, + resolvedProgram(), [], - [], - new ArrayBuffer(8), ); expect(worker.tcpListenerTargets.get(8080)).toContainEqual({ @@ -1036,6 +1078,18 @@ function createWorker(overrides: Record): any { }); } +function resolvedProgram() { + const programBytes = Uint8Array.from([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + ]).buffer; + return { + programBytes, + programModule: new WebAssembly.Module(programBytes), + argv: [], + }; +} + function createChannel(pid: number, memory: WebAssembly.Memory, channelOffset: number): any { return { pid, diff --git a/host/test/exec.test.ts b/host/test/exec.test.ts index 5726624ada..34743b9798 100644 --- a/host/test/exec.test.ts +++ b/host/test/exec.test.ts @@ -2,6 +2,9 @@ * Tests for execve support — loading a new program binary into an existing process. */ import { describe, it, expect } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { runCentralizedProgram } from "./centralized-test-helper"; import { tryResolveBinary } from "../src/binary-resolver"; @@ -59,4 +62,35 @@ describe("execve", () => { expect(result.stdout).toContain("argv[1]=from-fork"); expect(result.stdout).toContain("FROM=fork"); }); + + it.skipIf(!hasExecCaller)("rejects malformed Wasm before committing exec", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "kandelo-exec-malformed-wasm-")); + const malformedWasm = join(tempDir, "malformed.wasm"); + try { + // Valid Wasm magic/version with a truncated type section. This reaches + // compilation, which must fail before kernelExecSetup discards the old + // process image. + writeFileSync(malformedWasm, Buffer.from([ + 0x00, 0x61, 0x73, 0x6d, + 0x01, 0x00, 0x00, 0x00, + 0x01, 0x01, 0xff, + ])); + + const result = await runCentralizedProgram({ + programPath: execCallerBinary!, + argv: ["exec-caller"], + timeout: 15_000, + execPrograms: new Map([ + ["/bin/exec-child", malformedWasm], + ]), + }); + + expect(result.exitCode).toBe(127); + expect(result.stderr).toContain("Exec format error"); + expect(result.stderr).not.toContain("Centralized worker failed"); + expect(result.stderr).not.toContain("WebAssembly.compile()"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); }); diff --git a/host/test/fetch-backend.test.ts b/host/test/fetch-backend.test.ts index 5480c27dd7..21610509a4 100644 --- a/host/test/fetch-backend.test.ts +++ b/host/test/fetch-backend.test.ts @@ -139,9 +139,9 @@ describe("FetchNetworkBackend", () => { expect(() => backend.getaddrinfo(`www.${"x".repeat(100)}.com`)).toThrow("ENOENT"); }); - it("rejects names the browser resolver cannot truthfully synthesize", () => { + it("rejects the reserved invalid zone without rejecting unqualified names", () => { const backend = new FetchNetworkBackend(); - expect(() => backend.getaddrinfo("dummy-host-name")).toThrow("ENOENT"); + expect(backend.getaddrinfo("dummy-host-name").length).toBe(4); expect(() => backend.getaddrinfo("totes.invalid")).toThrow("ENOENT"); }); @@ -269,9 +269,9 @@ describe("TlsNetworkBackend HTTP proxy path", () => { expect(() => backend.getaddrinfo(`www.${"x".repeat(100)}.com`)).toThrow("ENOENT"); }); - it("rejects special-use invalid and unqualified names", () => { + it("rejects special-use invalid but permits potentially resolvable unqualified names", () => { const backend = new TlsNetworkBackend(); - expect(() => backend.getaddrinfo("dummy-host-name")).toThrow("ENOENT"); + expect(backend.getaddrinfo("dummy-host-name").length).toBe(4); expect(() => backend.getaddrinfo("totes.invalid")).toThrow("ENOENT"); }); @@ -333,6 +333,30 @@ describe("TlsNetworkBackend HTTP proxy path", () => { expect(response.toLowerCase()).not.toContain("connection: close"); }); + it("routes HTTP fetches through the configured CORS proxy", async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response("proxied")); + vi.stubGlobal("fetch", fetchMock); + const proxyPrefix = "https://kandelo.test/proxy?url="; + const backend = new TlsNetworkBackend({ + corsProxyUrl: proxyPrefix, + dnsAliases: {}, + }); + const addr = backend.getaddrinfo("example.com"); + backend.connect(1, addr, 80); + + backend.send( + 1, + encoder.encode("GET /resource HTTP/1.1\r\nHost: example.com\r\n\r\n"), + 0, + ); + await recvWhenReady(backend, 1); + + expect(fetchMock).toHaveBeenCalledWith( + `${proxyPrefix}${encodeURIComponent("http://example.com/resource")}`, + expect.any(Object), + ); + }); + it("honors MSG_PEEK without consuming HTTP response bytes", async () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("peek-body"))); const backend = new TlsNetworkBackend(); @@ -357,7 +381,7 @@ describe("TlsNetworkBackend HTTP proxy path", () => { }); describe("TlsNetworkBackend TLS MITM path", () => { - it("delivers the full response to recv() before reporting EOF", async () => { + it("polls and peeks encrypted response bytes before reporting EOF", async () => { const body = "mitm-response-body"; vi.stubGlobal( "fetch", @@ -380,8 +404,39 @@ describe("TlsNetworkBackend TLS MITM path", () => { .getWriter() .write(encoder.encode("GET /readme HTTP/1.1\r\nHost: example.com\r\n\r\n")); - const response = decoder.decode(await recvWhenReady(backend, 1)); + await waitForReadable(backend, 1); + const peeked = backend.recv(1, 8, MSG_PEEK); + expect(backend.poll(1, 0x0001) & 0x0001).toBe(0x0001); + const consumed = backend.recv(1, 8, 0); + expect(peeked).toEqual(consumed); + const response = decoder.decode( + new Uint8Array([...consumed, ...await recvWhenReady(backend, 1)]), + ); expect(response).toContain("200"); expect(response).toContain(body); }); + + it("routes decrypted HTTPS requests through the configured CORS proxy", async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response("proxied TLS")); + vi.stubGlobal("fetch", fetchMock); + const proxyPrefix = "https://kandelo.test/proxy?"; + let tls!: LoopbackMitmTls; + const backend = new TlsNetworkBackend({ + corsProxyUrl: proxyPrefix, + createTlsConnection: () => (tls = new LoopbackMitmTls()), + }); + await backend.init(); + + const addr = backend.getaddrinfo("example.com"); + backend.connect(2, addr, 443); + await tls.serverEnd.upstream.writable + .getWriter() + .write(encoder.encode("GET /secure HTTP/1.1\r\nHost: example.com\r\n\r\n")); + await waitForReadable(backend, 2); + + expect(fetchMock).toHaveBeenCalledWith( + `${proxyPrefix}https://example.com/secure`, + expect.any(Object), + ); + }); }); diff --git a/host/test/global-setup.ts b/host/test/global-setup.ts index bef9ff1ea4..f512821e54 100644 --- a/host/test/global-setup.ts +++ b/host/test/global-setup.ts @@ -21,6 +21,7 @@ const fixturesDir = join(__dirname, "fixtures"); /** C programs that tests depend on. */ const TEST_PROGRAMS = [ + "clock_getcpuclockid_test.c", "putenv_test.c", "getaddrinfo_test.c", "sysv_ipc_test.c", diff --git a/host/test/host-diagnostic-routing.test.ts b/host/test/host-diagnostic-routing.test.ts new file mode 100644 index 0000000000..ae7662aefd --- /dev/null +++ b/host/test/host-diagnostic-routing.test.ts @@ -0,0 +1,34 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "../.."); + +const entries = [ + ["Node", join(repoRoot, "host/src/node-kernel-worker-entry.ts")], + ["browser", join(repoRoot, "host/src/browser-kernel-worker-entry.ts")], +] as const; + +describe.each(entries)("%s kernel-worker diagnostic routing", (_name, path) => { + const source = readFileSync(path, "utf8"); + + it("reserves stderr protocol messages for the kernel's real onStderr bytes", () => { + const stderrPosts = source.match(/type:\s*"stderr"/g) ?? []; + expect(stderrPosts).toHaveLength(1); + expect(source).toMatch(/onStderr:[\s\S]{0,160}type:\s*"stderr"/); + }); + + it("routes lifecycle, protocol, exec, clone, and thread failures as host diagnostics", () => { + for (const diagnosticSource of [ + "worker protocol", + "worker-main error message", + "exec post-commit transition", + "clone allocation", + "thread worker failure", + ]) { + expect(source).toContain(`source: "${diagnosticSource}"`); + } + expect(source).toContain("reportHostDiagnostic({"); + }); +}); diff --git a/host/test/ifhwaddr.test.ts b/host/test/ifhwaddr.test.ts index c5ff39a534..e71e0c3cb4 100644 --- a/host/test/ifhwaddr.test.ts +++ b/host/test/ifhwaddr.test.ts @@ -1,33 +1,61 @@ import { describe, it, expect } from "vitest"; import { runCentralizedProgram } from "./centralized-test-helper"; import { resolveBinary } from "../src/binary-resolver"; +import { LocalVirtualNetwork } from "../src/networking/virtual-network"; +import { NodePlatformIO } from "../src/platform/node"; -describe("SIOCGIFCONF / SIOCGIFHWADDR", () => { - it("returns a virtual MAC address via ioctl", async () => { - const result = await runCentralizedProgram({ - programPath: resolveBinary("programs/ifhwaddr.wasm"), - timeout: 10_000, - }); +describe("virtual network interface ioctls", () => { + it.each([ + { + arch: "wasm32", + programPath: "programs/ifhwaddr.wasm", + ifreqSize: 32, + }, + { + arch: "wasm64", + programPath: "programs/wasm64/ifhwaddr.wasm", + ifreqSize: 40, + }, + ])( + "honors interface and guest-memory contracts for $arch", + async ({ arch, programPath, ifreqSize }) => { + const network = new LocalVirtualNetwork(); + const io = new NodePlatformIO(); + io.network = network.attachMachine({ + id: `ifhwaddr-${arch}`, + address: [10, 23, 45, 67], + }); - expect(result.exitCode).toBe(0); + const result = await runCentralizedProgram({ + programPath: resolveBinary(programPath), + io, + timeout: 30_000, + }); - // Should find one interface named "eth0" - expect(result.stdout).toContain("interfaces: 1"); - expect(result.stdout).toContain("name: eth0"); - expect(result.stdout).toContain("index: 1"); - expect(result.stdout).toContain("index-name: eth0"); + expect(result.exitCode).toBe(0); + expect(result.stdout).not.toContain("FAIL:"); + expect(result.stdout).toContain(`ifreq-size: ${ifreqSize}`); + expect(result.stdout).toContain("ifconf: lo=127.0.0.1 eth0=10.23.45.67"); + expect(result.stdout).toContain("eth0-address: 10.23.45.67"); + expect(result.stdout).toContain( + "libc-name-to-index: lo=1 eth0=2 missing=0 errno=19", + ); + expect(result.stdout).toContain("libc-invalid-index: errno=6"); + expect(result.stdout).toContain("nameindex: lo=1"); + expect(result.stdout).toContain("nameindex: eth0=2"); + expect(result.stdout).toContain( + "PASS: virtual interface ioctl and libc contracts", + ); - // MAC should be locally-administered and non-zero - expect(result.stdout).toContain("locally-administered: yes"); - expect(result.stdout).toContain("non-zero: yes"); + const macMatch = result.stdout.match( + /eth0-mac: ([0-9a-f]{2}(?::[0-9a-f]{2}){5})/, + ); + expect(macMatch).not.toBeNull(); - // MAC format: xx:xx:xx:xx:xx:xx - const macMatch = result.stdout.match(/mac: ([0-9a-f]{2}(?::[0-9a-f]{2}){5})/); - expect(macMatch).not.toBeNull(); - - // Verify locally-administered bit (second-lowest bit of first octet) - const firstOctet = parseInt(macMatch![1].split(":")[0], 16); - expect(firstOctet & 0x02).toBe(0x02); // locally administered - expect(firstOctet & 0x01).toBe(0x00); // unicast - }); + const firstOctet = parseInt(macMatch![1].split(":")[0], 16); + expect(firstOctet & 0x02).toBe(0x02); + expect(firstOctet & 0x01).toBe(0x00); + }, + 30_000, + ); }); diff --git a/host/test/node-kernel-host-diagnostic.test.ts b/host/test/node-kernel-host-diagnostic.test.ts new file mode 100644 index 0000000000..0440e45eed --- /dev/null +++ b/host/test/node-kernel-host-diagnostic.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from "vitest"; +import { NodeKernelHost } from "../src/node-kernel-host"; +import type { KernelToMainMessage } from "../src/node-kernel-protocol"; + +interface TestableNodeKernelHost { + handleWorkerMessage(message: KernelToMainMessage): void; +} + +describe("NodeKernelHost diagnostics", () => { + it("delivers host diagnostics without invoking the guest stderr callback", () => { + const onHostDiagnostic = vi.fn(); + const onStderr = vi.fn(); + const host = new NodeKernelHost({ onHostDiagnostic, onStderr }); + const testable = host as unknown as TestableNodeKernelHost; + + testable.handleWorkerMessage({ + type: "host_diagnostic", + pid: 42, + source: "clone allocation", + message: "host allocation failed", + }); + + expect(onHostDiagnostic).toHaveBeenCalledOnce(); + expect(onHostDiagnostic).toHaveBeenCalledWith({ + pid: 42, + source: "clone allocation", + message: "host allocation failed", + }); + expect(onStderr).not.toHaveBeenCalled(); + }); + + it("keeps actual stderr messages on the guest callback", () => { + const onHostDiagnostic = vi.fn(); + const onStderr = vi.fn(); + const host = new NodeKernelHost({ onHostDiagnostic, onStderr }); + const testable = host as unknown as TestableNodeKernelHost; + const data = new TextEncoder().encode("guest stderr\n"); + + testable.handleWorkerMessage({ type: "stderr", pid: 7, data }); + + expect(onStderr).toHaveBeenCalledWith(7, data); + expect(onHostDiagnostic).not.toHaveBeenCalled(); + }); +}); diff --git a/host/test/pthread-trap-semantics.test.ts b/host/test/pthread-trap-semantics.test.ts index 7cd6b76d31..6105928e12 100644 --- a/host/test/pthread-trap-semantics.test.ts +++ b/host/test/pthread-trap-semantics.test.ts @@ -30,7 +30,7 @@ describe.skipIf(!hasPthreadTrapBinaries)("pthread trap POSIX semantics", () => { }, 15_000); it("terminates the process when a pthread worker hits an uncaught guest trap", async () => { - const { exitCode, stderr } = await runCentralizedProgram({ + const { exitCode, stderr, hostDiagnostics } = await runCentralizedProgram({ programPath: trapChildBinary, argv: ["pthread-trap-child"], timeout: 10_000, @@ -38,7 +38,9 @@ describe.skipIf(!hasPthreadTrapBinaries)("pthread trap POSIX semantics", () => { }); expect(stderr).toContain("pthread-trap-child: before trap"); - expect(stderr).toContain("Thread worker failed"); + expect(stderr).not.toContain("Thread worker failed"); + expect(hostDiagnostics.map((entry) => entry.message).join("\n")) + .toContain("Thread worker failed"); expect(stderr).not.toContain("FAIL pthread_join returned"); expect(exitCode).toBe(signalExitStatus(SIGILL)); }, 15_000); diff --git a/host/test/tcp-backend.test.ts b/host/test/tcp-backend.test.ts index cda08a9be8..83f2fddb89 100644 --- a/host/test/tcp-backend.test.ts +++ b/host/test/tcp-backend.test.ts @@ -3,6 +3,8 @@ import * as net from "node:net"; import { TcpNetworkBackend } from "../src/networking/tcp-backend"; const LOOPBACK = new Uint8Array([127, 0, 0, 1]); +const POLLIN = 0x0001; +const MSG_PEEK = 0x0002; async function listenLoopback(): Promise<{ server: net.Server; @@ -48,6 +50,15 @@ async function waitForConnected(backend: TcpNetworkBackend, handle: number): Pro throw new Error("connect timed out"); } +async function waitForReadable(backend: TcpNetworkBackend, handle: number): Promise { + const deadline = Date.now() + 2_000; + while (Date.now() < deadline) { + if ((backend.poll(handle, POLLIN) & POLLIN) !== 0) return; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error("readable data timed out"); +} + describe("TcpNetworkBackend hostname parsing", () => { it.each([ ["2130706433", [127, 0, 0, 1]], @@ -94,6 +105,35 @@ describe("TcpNetworkBackend", () => { await expect(accepted).resolves.toEqual({ data: "hello", ended: true }); }); + + it("honors MSG_PEEK on real Node TCP sockets", async () => { + let acceptedSocket!: net.Socket; + let resolveAccepted!: () => void; + const accepted = new Promise((resolve) => { resolveAccepted = resolve; }); + const server = net.createServer((socket) => { + acceptedSocket = socket; + resolveAccepted(); + }); + servers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + + const backend = new TcpNetworkBackend(); + backend.connect(10, LOOPBACK, (server.address() as net.AddressInfo).port); + await waitForConnected(backend, 10); + await accepted; + acceptedSocket.write("peek-data"); + await waitForReadable(backend, 10); + + expect(new TextDecoder().decode(backend.recv(10, 4, MSG_PEEK))).toBe("peek"); + expect(new TextDecoder().decode(backend.recv(10, 9, 0))).toBe("peek-data"); + + backend.close(10); + acceptedSocket.destroy(); + }); + it("keeps the real writable half open after peer FIN", async () => { let acceptedSocket!: net.Socket; let resolveAccepted!: () => void; diff --git a/host/test/time-provider.test.ts b/host/test/time-provider.test.ts new file mode 100644 index 0000000000..cf26053cf7 --- /dev/null +++ b/host/test/time-provider.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { NodePlatformIO } from "../src/platform/node"; +import { BrowserTimeProvider, NodeTimeProvider } from "../src/vfs/time"; + +function asNanoseconds(value: { sec: number; nsec: number }): bigint { + return BigInt(value.sec) * 1_000_000_000n + BigInt(value.nsec); +} + +describe.each([ + ["NodeTimeProvider", () => new NodeTimeProvider()], + ["BrowserTimeProvider", () => new BrowserTimeProvider()], + ["NodePlatformIO", () => new NodePlatformIO()], +] as const)("%s CLOCK_BOOTTIME", (_name, createProvider) => { + it("uses the same nondecreasing domain as CLOCK_MONOTONIC", () => { + const provider = createProvider(); + const monotonic = asNanoseconds(provider.clockGettime(1)); + const boottime = asNanoseconds(provider.clockGettime(7)); + + expect(boottime).toBeGreaterThanOrEqual(monotonic); + expect(boottime - monotonic).toBeLessThan(100_000_000n); + }); +}); diff --git a/host/test/vm-interrupt-timer.test.ts b/host/test/vm-interrupt-timer.test.ts new file mode 100644 index 0000000000..f5b78af108 --- /dev/null +++ b/host/test/vm-interrupt-timer.test.ts @@ -0,0 +1,251 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + MAX_VM_INTERRUPT_TIMER_DELAY_MS, + VmInterruptTimerManager, + type VmInterruptTimerScheduler, +} from "../src/vm-interrupt-timer"; + +interface Generation { + memory: WebAssembly.Memory; +} + +function generation(shared = true): Generation { + return { + memory: new WebAssembly.Memory({ + initial: 1, + maximum: 2, + shared, + }), + }; +} + +function flag(generation: Generation, ptr: number): number { + return Atomics.load(new Uint8Array(generation.memory.buffer), ptr); +} + +describe("VmInterruptTimerManager", () => { + let nowMs: number; + let current: Map; + let delays: number[]; + let scheduler: VmInterruptTimerScheduler; + let manager: VmInterruptTimerManager; + + beforeEach(() => { + vi.useFakeTimers(); + nowMs = 0; + current = new Map(); + delays = []; + scheduler = { + now: () => nowMs, + set: (callback, delayMs) => { + delays.push(delayMs); + return setTimeout(callback, delayMs); + }, + clear: (handle) => clearTimeout(handle), + }; + manager = new VmInterruptTimerManager((pid) => current.get(pid), scheduler); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + function advance(ms: number): void { + nowMs += ms; + vi.advanceTimersByTime(ms); + } + + it("sets both flags at the exact monotonic deadline", () => { + const process = generation(); + current.set(41, process); + + expect(manager.arm(41, process, { + timedOutPtr: 12, + vmInterruptPtr: 13, + seconds: 1, + })).toBe(true); + + advance(999); + expect(flag(process, 12)).toBe(0); + expect(flag(process, 13)).toBe(0); + + advance(1); + expect(flag(process, 12)).toBe(1); + expect(flag(process, 13)).toBe(1); + expect(manager.activeCount).toBe(0); + }); + + it("re-arms from the new request time and suppresses the old callback", () => { + const process = generation(); + current.set(42, process); + + manager.arm(42, process, { + timedOutPtr: 20, + vmInterruptPtr: 21, + seconds: 1, + }); + advance(500); + manager.arm(42, process, { + timedOutPtr: 20, + vmInterruptPtr: 21, + seconds: 2, + }); + + advance(500); + expect(flag(process, 20)).toBe(0); + advance(1_499); + expect(flag(process, 20)).toBe(0); + advance(1); + expect(flag(process, 20)).toBe(1); + }); + + it("cancels through a non-positive runtime-hook request", () => { + const process = generation(); + current.set(43, process); + manager.arm(43, process, { + timedOutPtr: 24, + vmInterruptPtr: 25, + seconds: 1, + }); + + expect(manager.handleRequest(43, process, { + timedOutPtr: 24, + vmInterruptPtr: 25, + seconds: 0, + })).toBe(true); + advance(1_000); + + expect(flag(process, 24)).toBe(0); + expect(flag(process, 25)).toBe(0); + expect(manager.activeCount).toBe(0); + }); + + it("refuses a stale process generation without disturbing its replacement", () => { + const oldProcess = generation(); + const newProcess = generation(); + current.set(44, oldProcess); + manager.arm(44, oldProcess, { + timedOutPtr: 28, + vmInterruptPtr: 29, + seconds: 1, + }); + + current.set(44, newProcess); + manager.arm(44, newProcess, { + timedOutPtr: 30, + vmInterruptPtr: 31, + seconds: 1, + }); + expect(manager.cancel(44, oldProcess)).toBe(false); + + advance(1_000); + expect(flag(oldProcess, 28)).toBe(0); + expect(flag(newProcess, 30)).toBe(1); + expect(flag(newProcess, 31)).toBe(1); + }); + + it("drops a queued callback when the PID generation changes", () => { + const oldProcess = generation(); + const replacement = generation(); + current.set(45, oldProcess); + manager.arm(45, oldProcess, { + timedOutPtr: 32, + vmInterruptPtr: 33, + seconds: 1, + }); + + current.set(45, replacement); + advance(1_000); + + expect(flag(oldProcess, 32)).toBe(0); + expect(flag(replacement, 32)).toBe(0); + expect(manager.activeCount).toBe(0); + }); + + it("chunks delays above the JavaScript signed-32-bit timer limit", () => { + const process = generation(); + current.set(46, process); + const tailMs = 2_500; + manager.arm(46, process, { + timedOutPtr: 36, + vmInterruptPtr: 37, + seconds: (MAX_VM_INTERRUPT_TIMER_DELAY_MS + tailMs) / 1_000, + }); + + expect(delays).toEqual([MAX_VM_INTERRUPT_TIMER_DELAY_MS]); + advance(MAX_VM_INTERRUPT_TIMER_DELAY_MS); + expect(flag(process, 36)).toBe(0); + expect(delays).toEqual([MAX_VM_INTERRUPT_TIMER_DELAY_MS, tailMs]); + + advance(tailMs - 1); + expect(flag(process, 36)).toBe(0); + advance(1); + expect(flag(process, 36)).toBe(1); + }); + + it("rejects out-of-bounds and non-shared flag storage", () => { + const process = generation(); + current.set(47, process); + expect(manager.arm(47, process, { + timedOutPtr: process.memory.buffer.byteLength, + vmInterruptPtr: 1, + seconds: 1, + })).toBe(false); + + const unshared = generation(false); + current.set(48, unshared); + expect(manager.arm(48, unshared, { + timedOutPtr: 1, + vmInterruptPtr: 2, + seconds: 1, + })).toBe(false); + expect(manager.activeCount).toBe(0); + }); + + it("clears every process timer", () => { + const first = generation(); + const second = generation(); + current.set(49, first); + current.set(50, second); + manager.arm(49, first, { timedOutPtr: 40, vmInterruptPtr: 41, seconds: 1 }); + manager.arm(50, second, { timedOutPtr: 42, vmInterruptPtr: 43, seconds: 1 }); + + manager.clearAll(); + advance(1_000); + + expect(flag(first, 40)).toBe(0); + expect(flag(second, 42)).toBe(0); + expect(manager.activeCount).toBe(0); + }); + + it("cancels a timer whose scheduler handle is numeric zero", () => { + const process = generation(); + const callbacks: Array<() => void> = []; + const cleared: number[] = []; + const zeroHandleScheduler: VmInterruptTimerScheduler = { + now: () => 0, + set: (callback) => { + callbacks.push(callback); + return 0; + }, + clear: (handle) => cleared.push(handle), + }; + current.set(51, process); + const zeroManager = new VmInterruptTimerManager( + (pid) => current.get(pid), + zeroHandleScheduler, + ); + + zeroManager.arm(51, process, { + timedOutPtr: 44, + vmInterruptPtr: 45, + seconds: 1, + }); + expect(zeroManager.cancel(51, process)).toBe(true); + expect(cleared).toEqual([0]); + + callbacks[0](); + expect(flag(process, 44)).toBe(0); + expect(flag(process, 45)).toBe(0); + }); +}); diff --git a/host/test/wasm-trap.test.ts b/host/test/wasm-trap.test.ts index f48a0a346f..4644509244 100644 --- a/host/test/wasm-trap.test.ts +++ b/host/test/wasm-trap.test.ts @@ -45,7 +45,7 @@ const programsBuilt = existsSync(wasmTrapBin) && describe.skipIf(!programsBuilt)("wasm trap → host exit (regression)", () => { it("__builtin_trap() in user code: spawn() resolves promptly, no hang", async () => { const t0 = Date.now(); - const { exitCode, stderr } = await runCentralizedProgram({ + const { exitCode, stderr, hostDiagnostics } = await runCentralizedProgram({ programPath: wasmTrapBin, useDefaultRootfs: false, timeout: 5000, @@ -54,17 +54,19 @@ describe.skipIf(!programsBuilt)("wasm trap → host exit (regression)", () => { expect(stderr).toContain("before-trap"); expect(stderr).not.toContain("SHOULD-NEVER-REACH"); + expect(stderr).not.toContain("RuntimeError"); // Arbitrary `unreachable` traps are no longer masked as successful exits; // only the known kernel_exit path is interpreted as normal termination. expect(exitCode).toBe(signalExitStatus(SIGILL)); - expect(stderr).toContain("RuntimeError"); - expect(stderr).toContain("unreachable"); + const diagnosticText = hostDiagnostics.map((entry) => entry.message).join("\n"); + expect(diagnosticText).toContain("RuntimeError"); + expect(diagnosticText).toContain("unreachable"); expect(elapsed).toBeLessThan(3000); }, 8_000); it("out-of-bounds memory trap resolves as SIGSEGV", async () => { const t0 = Date.now(); - const { exitCode, stderr } = await runCentralizedProgram({ + const { exitCode, stderr, hostDiagnostics } = await runCentralizedProgram({ programPath: oobTrapBin, useDefaultRootfs: false, timeout: 5000, @@ -73,14 +75,15 @@ describe.skipIf(!programsBuilt)("wasm trap → host exit (regression)", () => { expect(stderr).toContain("before-oob"); expect(stderr).not.toContain("SHOULD-NEVER-REACH"); + expect(stderr).not.toContain("RuntimeError"); expect(exitCode).toBe(signalExitStatus(SIGSEGV)); - expect(stderr).toContain("RuntimeError"); + expect(hostDiagnostics.map((entry) => entry.message).join("\n")).toContain("RuntimeError"); expect(elapsed).toBeLessThan(3000); }, 8_000); it("integer divide-by-zero trap resolves as SIGFPE", async () => { const t0 = Date.now(); - const { exitCode, stderr } = await runCentralizedProgram({ + const { exitCode, stderr, hostDiagnostics } = await runCentralizedProgram({ programPath: divzeroTrapBin, useDefaultRootfs: false, timeout: 5000, @@ -89,8 +92,9 @@ describe.skipIf(!programsBuilt)("wasm trap → host exit (regression)", () => { expect(stderr).toContain("before-divzero"); expect(stderr).not.toContain("SHOULD-NEVER-REACH"); + expect(stderr).not.toContain("RuntimeError"); expect(exitCode).toBe(signalExitStatus(SIGFPE)); - expect(stderr).toContain("RuntimeError"); + expect(hostDiagnostics.map((entry) => entry.message).join("\n")).toContain("RuntimeError"); expect(elapsed).toBeLessThan(3000); }, 8_000); diff --git a/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c b/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c index 2e2d3a462e..6293689f6b 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_indextoname.c @@ -1,11 +1,23 @@ +#define _GNU_SOURCE #include +#include +#include #include #include +#include "syscall.h" char *if_indextoname(unsigned index, char *name) { - if (index == 1) - return strncpy(name, "eth0", IF_NAMESIZE); - errno = ENXIO; - return 0; + struct ifreq ifr = {0}; + int fd, r; + + if ((fd = socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) return 0; + ifr.ifr_ifindex = index; + r = ioctl(fd, SIOCGIFNAME, &ifr); + __syscall(SYS_close, fd); + if (r < 0) { + if (errno == ENODEV) errno = ENXIO; + return 0; + } + return strncpy(name, ifr.ifr_name, IF_NAMESIZE); } diff --git a/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c b/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c index 6676a28831..5ff17a8b9d 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_nameindex.c @@ -1,19 +1,49 @@ +#define _GNU_SOURCE #include +#include +#include #include #include +#include "syscall.h" struct if_nameindex *if_nameindex(void) { - /* Return the kernel's synthetic virtual network interface. */ - struct if_nameindex *idx = malloc(2 * sizeof(*idx)); - if (!idx) return 0; - idx[0].if_index = 1; - idx[0].if_name = strdup("eth0"); - if (!idx[0].if_name) { - free(idx); - return 0; + struct ifconf ifc = {0}; + struct ifreq *req = 0; + struct if_nameindex *idx = 0; + size_t count = 0; + int fd; + + if ((fd = socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) return 0; + + /* Query the host-visible interface list instead of embedding interface + * numbers in libc. The terminating zero entry comes from calloc. */ + if (ioctl(fd, SIOCGIFCONF, &ifc) < 0 || ifc.ifc_len < 0) goto fail; + if (ifc.ifc_len) { + req = malloc(ifc.ifc_len); + if (!req) goto fail; + ifc.ifc_req = req; + if (ioctl(fd, SIOCGIFCONF, &ifc) < 0) goto fail; + count = ifc.ifc_len / sizeof(*req); + } + + idx = calloc(count + 1, sizeof(*idx)); + if (!idx) goto fail; + + for (size_t i = 0; i < count; i++) { + if (ioctl(fd, SIOCGIFINDEX, &req[i]) < 0) goto fail; + idx[i].if_index = req[i].ifr_ifindex; + idx[i].if_name = strdup(req[i].ifr_name); + if (!idx[i].if_name) goto fail; } - idx[1].if_index = 0; - idx[1].if_name = 0; + __syscall(SYS_close, fd); + free(req); return idx; + +fail: + __syscall(SYS_close, fd); + for (size_t i = 0; i < count; i++) free(idx ? idx[i].if_name : 0); + free(idx); + free(req); + return 0; } diff --git a/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c b/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c index 0141c18e31..540da56d7b 100644 --- a/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c +++ b/libc/musl-overlay/src/network/wasm32posix/if_nametoindex.c @@ -1,8 +1,18 @@ +#define _GNU_SOURCE #include +#include +#include #include +#include "syscall.h" unsigned if_nametoindex(const char *name) { - if (!strcmp(name, "eth0")) return 1; - return 0; + struct ifreq ifr = {0}; + int fd, r; + + if ((fd = socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) return 0; + strncpy(ifr.ifr_name, name, sizeof ifr.ifr_name - 1); + r = ioctl(fd, SIOCGIFINDEX, &ifr); + __syscall(SYS_close, fd); + return r < 0 ? 0 : ifr.ifr_ifindex; } diff --git a/programs/ifhwaddr.c b/programs/ifhwaddr.c index 8bf2072d44..00143f6f22 100644 --- a/programs/ifhwaddr.c +++ b/programs/ifhwaddr.c @@ -1,6 +1,8 @@ -/* Test program: retrieve network interface hardware (MAC) address via ioctl. - * Uses SIOCGIFCONF to enumerate interfaces, then SIOCGIFHWADDR on each. */ +/* Exercise Kandelo's virtual-interface ioctl and libc name/index contracts. */ +#include +#include #include +#include #include #include #include @@ -8,59 +10,278 @@ #include #include -int main(void) { - struct ifreq ifr[8]; - struct ifconf ifc; - int fd, i; +static int failures; - fd = socket(AF_INET, SOCK_DGRAM, 0); - if (fd < 0) { - perror("socket"); - return 1; +static void check(int condition, const char *description) +{ + if (!condition) { + printf("FAIL: %s (errno=%d)\n", description, errno); + failures++; } +} - ifc.ifc_req = ifr; - ifc.ifc_len = sizeof(ifr); +static void set_ifreq_name(struct ifreq *ifr, const char *name) +{ + memset(ifr, 0, sizeof(*ifr)); + strncpy(ifr->ifr_name, name, IF_NAMESIZE - 1); +} - if (ioctl(fd, SIOCGIFCONF, &ifc) < 0) { - perror("SIOCGIFCONF"); - close(fd); - return 1; +static unsigned char *ifreq_ipv4(struct ifreq *ifr) +{ + return (unsigned char *)&ifr->ifr_addr + 4; +} + +static int bytes_equal(const unsigned char *actual, + unsigned char a, unsigned char b, + unsigned char c, unsigned char d) +{ + return actual[0] == a && actual[1] == b && + actual[2] == c && actual[3] == d; +} + +static void check_ifconf(int fd) +{ + struct ifconf ifc = {0}; + struct ifreq interfaces[4]; + + check(ioctl(fd, SIOCGIFCONF, &ifc) == 0, + "SIOCGIFCONF size query succeeds"); + check(ifc.ifc_len == 2 * (int)sizeof(struct ifreq), + "SIOCGIFCONF size query reports two interfaces"); + printf("ifreq-size: %zu\n", sizeof(struct ifreq)); + + struct { + struct ifreq entry; + unsigned char guard[16]; + } bounded; + memset(&bounded, 0, sizeof(bounded)); + memset(bounded.guard, 0xa5, sizeof(bounded.guard)); + ifc.ifc_len = sizeof(bounded.entry); + ifc.ifc_req = &bounded.entry; + check(ioctl(fd, SIOCGIFCONF, &ifc) == 0, + "bounded SIOCGIFCONF succeeds"); + check(ifc.ifc_len == (int)sizeof(struct ifreq), + "bounded SIOCGIFCONF writes one complete entry"); + check(strcmp(bounded.entry.ifr_name, "lo") == 0, + "bounded SIOCGIFCONF returns lo first"); + for (size_t i = 0; i < sizeof(bounded.guard); i++) { + check(bounded.guard[i] == 0xa5, + "bounded SIOCGIFCONF preserves trailing guard bytes"); } - int n = ifc.ifc_len / sizeof(struct ifreq); - printf("interfaces: %d\n", n); + memset(interfaces, 0xcc, sizeof(interfaces)); + ifc.ifc_len = sizeof(interfaces); + ifc.ifc_req = interfaces; + check(ioctl(fd, SIOCGIFCONF, &ifc) == 0, + "full SIOCGIFCONF succeeds"); + int count = ifc.ifc_len / (int)sizeof(struct ifreq); + check(count == 2, "full SIOCGIFCONF returns two interfaces"); + check(strcmp(interfaces[0].ifr_name, "lo") == 0, + "SIOCGIFCONF names loopback lo"); + check(strcmp(interfaces[1].ifr_name, "eth0") == 0, + "SIOCGIFCONF names external interface eth0"); + check(interfaces[0].ifr_addr.sa_family == AF_INET, + "lo SIOCGIFCONF address uses AF_INET"); + check(interfaces[1].ifr_addr.sa_family == AF_INET, + "eth0 SIOCGIFCONF address uses AF_INET"); + check(bytes_equal(ifreq_ipv4(&interfaces[0]), 127, 0, 0, 1), + "lo SIOCGIFCONF address is loopback"); + unsigned char *eth = ifreq_ipv4(&interfaces[1]); + printf("ifconf: lo=127.0.0.1 eth0=%u.%u.%u.%u\n", + eth[0], eth[1], eth[2], eth[3]); - for (i = 0; i < n; i++) { - printf("name: %s\n", ifr[i].ifr_name); - unsigned index = if_nametoindex(ifr[i].ifr_name); - printf("index: %u\n", index); - if (index != 0) { - char name[IF_NAMESIZE]; - printf("index-name: %s\n", if_indextoname(index, name) ? name : "(null)"); - } + ifc.ifc_len = sizeof(struct ifreq); + ifc.ifc_req = (struct ifreq *)(uintptr_t)-16; + errno = 0; + check(ioctl(fd, SIOCGIFCONF, &ifc) == -1 && errno == EFAULT, + "SIOCGIFCONF rejects an invalid nested buffer"); - if (ioctl(fd, SIOCGIFHWADDR, &ifr[i]) < 0) { - perror("SIOCGIFHWADDR"); - continue; - } + ifc.ifc_len = -1; + ifc.ifc_req = interfaces; + errno = 0; + check(ioctl(fd, SIOCGIFCONF, &ifc) == -1 && errno == EINVAL, + "SIOCGIFCONF rejects a negative length"); - unsigned char *mac = (unsigned char *)ifr[i].ifr_hwaddr.sa_data; - printf("mac: %02x:%02x:%02x:%02x:%02x:%02x\n", - mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + errno = 0; + check(ioctl(fd, SIOCGIFCONF, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFCONF rejects an invalid outer pointer"); +} - /* Check locally-administered bit */ - if (mac[0] & 0x02) { - printf("locally-administered: yes\n"); - } - /* Check non-zero */ - int all_zero = 1; - for (int j = 0; j < 6; j++) { - if (mac[j]) { all_zero = 0; break; } +static void check_name_index_ioctls(int fd) +{ + struct ifreq ifr; + + set_ifreq_name(&ifr, "lo"); + check(ioctl(fd, SIOCGIFINDEX, &ifr) == 0 && ifr.ifr_ifindex == 1, + "SIOCGIFINDEX maps lo to 1"); + set_ifreq_name(&ifr, "eth0"); + check(ioctl(fd, SIOCGIFINDEX, &ifr) == 0 && ifr.ifr_ifindex == 2, + "SIOCGIFINDEX maps eth0 to 2"); + set_ifreq_name(&ifr, "missing0"); + errno = 0; + check(ioctl(fd, SIOCGIFINDEX, &ifr) == -1 && errno == ENODEV, + "SIOCGIFINDEX rejects an unknown name"); + + memset(&ifr, 0, sizeof(ifr)); + ifr.ifr_ifindex = 1; + check(ioctl(fd, SIOCGIFNAME, &ifr) == 0 && + strcmp(ifr.ifr_name, "lo") == 0, + "SIOCGIFNAME maps 1 to lo"); + memset(&ifr, 0, sizeof(ifr)); + ifr.ifr_ifindex = 2; + check(ioctl(fd, SIOCGIFNAME, &ifr) == 0 && + strcmp(ifr.ifr_name, "eth0") == 0, + "SIOCGIFNAME maps 2 to eth0"); + memset(&ifr, 0, sizeof(ifr)); + ifr.ifr_ifindex = 99; + errno = 0; + check(ioctl(fd, SIOCGIFNAME, &ifr) == -1 && errno == ENODEV, + "SIOCGIFNAME rejects an unknown index"); + + errno = 0; + check(ioctl(fd, SIOCGIFINDEX, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFINDEX rejects an invalid pointer"); + errno = 0; + check(ioctl(fd, SIOCGIFNAME, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFNAME rejects an invalid pointer"); +} + +static void check_addresses(int fd) +{ + struct ifreq ifr; + + set_ifreq_name(&ifr, "lo"); + check(ioctl(fd, SIOCGIFADDR, &ifr) == 0, + "SIOCGIFADDR returns lo address"); + check(ifr.ifr_addr.sa_family == AF_INET && + bytes_equal(ifreq_ipv4(&ifr), 127, 0, 0, 1), + "SIOCGIFADDR reports 127.0.0.1 for lo"); + + set_ifreq_name(&ifr, "eth0"); + check(ioctl(fd, SIOCGIFADDR, &ifr) == 0, + "SIOCGIFADDR returns the backend eth0 address"); + check(ifr.ifr_addr.sa_family == AF_INET, + "eth0 SIOCGIFADDR address uses AF_INET"); + unsigned char *address = ifreq_ipv4(&ifr); + printf("eth0-address: %u.%u.%u.%u\n", + address[0], address[1], address[2], address[3]); + + set_ifreq_name(&ifr, "missing0"); + errno = 0; + check(ioctl(fd, SIOCGIFADDR, &ifr) == -1 && errno == ENODEV, + "SIOCGIFADDR rejects an unknown name"); + errno = 0; + check(ioctl(fd, SIOCGIFADDR, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFADDR rejects an invalid pointer"); +} + +static void check_hardware_addresses(int fd) +{ + struct ifreq ifr; + + set_ifreq_name(&ifr, "lo"); + check(ioctl(fd, SIOCGIFHWADDR, &ifr) == 0, + "SIOCGIFHWADDR returns lo hardware type"); + check(ifr.ifr_hwaddr.sa_family == ARPHRD_LOOPBACK, + "lo uses ARPHRD_LOOPBACK"); + int all_zero = 1; + for (int i = 0; i < 6; i++) { + if ((unsigned char)ifr.ifr_hwaddr.sa_data[i] != 0) all_zero = 0; + } + check(all_zero, "lo hardware address is all zero"); + + set_ifreq_name(&ifr, "eth0"); + check(ioctl(fd, SIOCGIFHWADDR, &ifr) == 0, + "SIOCGIFHWADDR returns eth0 hardware address"); + check(ifr.ifr_hwaddr.sa_family == ARPHRD_ETHER, + "eth0 uses ARPHRD_ETHER"); + unsigned char *mac = (unsigned char *)ifr.ifr_hwaddr.sa_data; + all_zero = 1; + for (int i = 0; i < 6; i++) if (mac[i]) all_zero = 0; + check(!all_zero, "eth0 hardware address is non-zero"); + check((mac[0] & 0x02) != 0 && (mac[0] & 0x01) == 0, + "eth0 hardware address is local unicast"); + printf("eth0-mac: %02x:%02x:%02x:%02x:%02x:%02x\n", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + + set_ifreq_name(&ifr, "missing0"); + errno = 0; + check(ioctl(fd, SIOCGIFHWADDR, &ifr) == -1 && errno == ENODEV, + "SIOCGIFHWADDR rejects an unknown name"); + errno = 0; + check(ioctl(fd, SIOCGIFHWADDR, (void *)(uintptr_t)-16) == -1 && + errno == EFAULT, + "SIOCGIFHWADDR rejects an invalid pointer"); +} + +static void check_libc_name_index(void) +{ + errno = 0; + unsigned lo = if_nametoindex("lo"); + unsigned eth0 = if_nametoindex("eth0"); + unsigned missing = if_nametoindex("missing0"); + int missing_errno = errno; + check(lo == 1 && eth0 == 2, "if_nametoindex uses host ioctl mappings"); + check(missing == 0 && missing_errno == ENODEV, + "if_nametoindex rejects an unknown name"); + printf("libc-name-to-index: lo=%u eth0=%u missing=%u errno=%d\n", + lo, eth0, missing, missing_errno); + + char name[IF_NAMESIZE]; + check(if_indextoname(1, name) && strcmp(name, "lo") == 0, + "if_indextoname maps 1 to lo"); + check(if_indextoname(2, name) && strcmp(name, "eth0") == 0, + "if_indextoname maps 2 to eth0"); + errno = 0; + char *invalid = if_indextoname(99, name); + int invalid_errno = errno; + check(!invalid && invalid_errno == ENXIO, + "if_indextoname reports ENXIO for an unknown index"); + printf("libc-invalid-index: errno=%d\n", invalid_errno); + + struct if_nameindex *list = if_nameindex(); + check(list != NULL, "if_nameindex returns an interface list"); + if (list) { + int count = 0; + int saw_lo = 0; + int saw_eth0 = 0; + for (struct if_nameindex *entry = list; entry->if_index; entry++) { + printf("nameindex: %s=%u\n", entry->if_name, entry->if_index); + saw_lo |= entry->if_index == 1 && + strcmp(entry->if_name, "lo") == 0; + saw_eth0 |= entry->if_index == 2 && + strcmp(entry->if_name, "eth0") == 0; + count++; } - printf("non-zero: %s\n", all_zero ? "no" : "yes"); + check(count == 2 && saw_lo && saw_eth0, + "if_nameindex discovers lo and eth0 through ioctls"); + if_freenameindex(list); + } +} + +int main(void) +{ + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) { + perror("socket"); + return 1; } + check_ifconf(fd); + check_name_index_ioctls(fd); + check_addresses(fd); + check_hardware_addresses(fd); + check_libc_name_index(); + close(fd); + if (failures) { + printf("FAILURES: %d\n", failures); + return 1; + } + printf("PASS: virtual interface ioctl and libc contracts\n"); return 0; } diff --git a/scripts/build-programs.sh b/scripts/build-programs.sh index 7fe98b59d5..36dfab67ab 100755 --- a/scripts/build-programs.sh +++ b/scripts/build-programs.sh @@ -264,9 +264,12 @@ if [ -f "$SYSROOT64/lib/libc.a" ]; then -Wl,--export=__tls_align -Wl,--export=__stack_pointer -Wl,--export=__wasm_thread_init + -Wl,--export=__abi_version ) - for src in "$REPO_ROOT/programs/"hello64.c; do + for src in \ + "$REPO_ROOT/programs/"hello64.c \ + "$REPO_ROOT/programs/"ifhwaddr.c; do [ -f "$src" ] || continue local_name=$(basename "$src" .c) echo " Compiling $local_name (wasm64)..." From 04b60aeb143a78c59afad6c680d58f5172f85764 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 07:48:34 -0400 Subject: [PATCH 24/29] fix: make SharedFS persistence and lazy backing race-safe Serialize namespace mutations, bound sparse growth, preserve partial-write and ENOSPC state, snapshot only quiescent filesystem data, bind deferred backing to inode generations and data sequences, and preserve hard-link identity during rebases. Performance was not measured. --- docs/architecture.md | 72 +- host/src/vfs/memory-fs.ts | 1109 +++++++++++++---- host/src/vfs/sharedfs-vendor.ts | 943 +++++++++++--- host/test/browser-kernel.test.ts | 96 +- .../fixtures/sharedfs-lock-release-worker.mjs | 10 + .../fixtures/sharedfs-namespace-worker.ts | 57 + host/test/lazy-archive.test.ts | 362 +++++- host/test/sharedfs-safety.test.ts | 937 ++++++++++++++ host/test/vfs-image.test.ts | 273 +++- host/test/vfs.test.ts | 78 +- packages/registry/erlang-vfs/build.toml | 2 +- packages/registry/kandelo-sdk/build.toml | 2 +- packages/registry/mariadb-test/build.toml | 2 +- packages/registry/mariadb-vfs/build.toml | 2 +- packages/registry/node-vfs/build.toml | 2 +- packages/registry/perl-vfs/build.toml | 2 +- packages/registry/python-vfs/build.toml | 2 +- 17 files changed, 3490 insertions(+), 461 deletions(-) create mode 100644 host/test/fixtures/sharedfs-lock-release-worker.mjs create mode 100644 host/test/fixtures/sharedfs-namespace-worker.ts create mode 100644 host/test/sharedfs-safety.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 858ad1b168..4afeea25e4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -123,6 +123,37 @@ Key host components: | NodeWorkerAdapter | `worker-adapter.ts` | Creates Node.js worker_threads | | BrowserWorkerAdapter | `worker-adapter-browser.ts` | Creates Web Workers | +### Cooperative process-runtime interrupts + +A process module may import +`env.__wasm_posix_vm_interrupt_after(timed_out_ptr, vm_interrupt_ptr, seconds)` +to request a cooperative runtime deadline. This import is intended for +language runtimes that already poll an interrupt flag at VM safepoints; it is +not a substitute for POSIX signal delivery or a general program-specific +kernel hook. + +The process worker forwards each arm or cancellation request to its host's +dedicated kernel worker. The kernel worker owns the JavaScript timer because a +process worker executing a CPU-bound Wasm loop cannot service its own event +loop. At the monotonic deadline the host atomically sets both bytes in the +process's shared memory. Timer entries retain the exact process-generation +object as well as the PID, so exec, exit, or PID reuse cannot redirect a stale +callback into a replacement process. Deadlines beyond JavaScript's signed +32-bit timer range are scheduled in bounded chunks. + +The imported function and its pointer-width-specific signature are part of +the guest/host ABI. A package that starts importing it must raise its kernel +ABI floor in the same ABI reconciliation that versions and snapshots the new +surface. + +Host-runtime lifecycle diagnostics are kept separate from guest file +descriptor 2. Both kernel-worker entries send typed `host_diagnostic` messages; +`NodeKernelHost` and `BrowserKernel` expose them through `onHostDiagnostic`. +Node also records them on the embedding process's console, while the browser +live-demo consumer records them in dmesg. Neither path appends host diagnostics +to the program's stderr byte stream, so test harnesses and applications observe +only bytes the guest actually wrote. + ### 3. Glue Layer (C) **Location**: `libc/glue/` @@ -499,7 +530,20 @@ const ino = mfs.registerLazyFile("/usr/bin/php", "https://cdn.example.com/php.wa await mfs.ensureMaterialized("/usr/bin/php"); ``` -Lazy file metadata (`path`, `url`, `size`, `ino`) can be transferred between instances via `exportLazyEntries()` / `importLazyEntries()` — used when forking workers that share the same SharedArrayBuffer. +Lazy file metadata (`path`, hard-link aliases, `url`, `size`, `ino`, inode +generation, and data-mutation sequence) can be transferred between instances +via `exportLazyEntries()` / `importLazyEntries()` — used when workers share the +same SharedArrayBuffer. The generation prevents an unlinked lazy inode from +transferring its URL or declared size to a later file that reuses the same +guest-visible inode number. The data sequence prevents an asynchronous fetch +from overwriting guest data written through another worker while the request +was in flight. Live cross-worker imports require both identity fields; only a +legacy image whose filesystem bytes and lazy JSON form one trusted artifact can +adopt older metadata, and only for an untouched empty stub. Filesystem rebasing +preserves hard-link identity for lazy and +concrete files rather than copying aliases into independent inodes. A rebase +walks one quiescent source snapshot, so a peer rename cannot mix lazy paths +from one namespace state with bytes from another. ### VFS Images @@ -515,6 +559,17 @@ const image: Uint8Array = await mfs.saveImage(); const fullImage: Uint8Array = await mfs.saveImage({ materializeAll: true }); ``` +Image creation is a quiescent filesystem operation. `saveImage()` rejects a +filesystem with live file or directory descriptors rather than serializing FD +tables, inode open-reference counts, or lock words as durable state. The +resulting bytes contain only filesystem state. Restore also clears those +runtime-only fields in legacy images, so a new machine never inherits handles +or locks from the image builder. Lazy-file and lazy-archive paths are collected +under the same namespace transaction as the filesystem bytes, including names +changed by another worker. `materializeAll: true` resolves both standalone and +archive-backed entries and fails instead of emitting an image that still +depends on a deferred URL. + **Restore from an image:** ```typescript @@ -561,11 +616,10 @@ Build scripts are in `images/vfs/scripts/` and share common helpers (`vfs-image- **Binary format:** -The on-disk file is the raw VFS image below, wrapped in a single zstd -frame. `saveImage()` always writes the compressed form (`.vfs.zst`); -`MemoryFileSystem.fromImage()` accepts either form and auto-detects -the zstd magic (`28 B5 2F FD`) at offset 0 to decide whether to -decompress before parsing. +`MemoryFileSystem.saveImage()` returns the raw VFS image below. The image +builder helper wraps it in one zstd frame for `.vfs.zst` artifacts; +`MemoryFileSystem.fromImage()` accepts either form and auto-detects the zstd +magic (`28 B5 2F FD`) at offset 0 before parsing. Decompressed layout: @@ -573,11 +627,13 @@ Decompressed layout: Offset Size Field 0 4 Magic: 0x56465349 ("VFSI") 4 4 Version: 1 -8 4 Flags: bit 0 = lazy entries included +8 4 Flags: bit 0 = lazy files, bit 1 = lazy archives, bit 2 = metadata 12 4 SharedArrayBuffer data length (N) 16 N Raw SharedArrayBuffer bytes (block filesystem) 16+N 4 Lazy entries JSON length (M) -20+N M Lazy entries as JSON (UTF-8): [{ino, path, url, size}, ...] +20+N M Lazy-file JSON (identity, aliases, URL, declared size) +... 4+L Optional lazy-archive JSON length and bytes (when bit 1 is set) +... 4+P Optional image-metadata JSON length and bytes (when bit 2 is set) ``` ## Networking diff --git a/host/src/vfs/memory-fs.ts b/host/src/vfs/memory-fs.ts index edc470bce0..aaa5239feb 100644 --- a/host/src/vfs/memory-fs.ts +++ b/host/src/vfs/memory-fs.ts @@ -4,6 +4,8 @@ import { SFFS_SUPER_MAGIC } from "../statfs"; import type { FileSystemBackend, DirEntry } from "./types"; import { SharedFS, + type NamespaceEntryIdentity, + type SharedFsIdentityState, type StatResult as SfsStatResult, } from "./sharedfs-vendor"; import type { ZipEntry } from "./zip"; @@ -11,7 +13,13 @@ import type { ZipEntry } from "./zip"; /** Serializable lazy file entry for transfer between instances. */ export interface LazyFileEntry { ino: number; + /** Inode-slot generation; omitted only by legacy serialized metadata. */ + generation?: number; + /** Inode data-mutation sequence; omitted only by legacy metadata. */ + dataSequence?: number; path: string; + /** All hard-link names for this inode; omitted by legacy metadata. */ + paths?: string[]; url: string; size: number; } @@ -37,9 +45,17 @@ export type LazyDownloadListener = (event: LazyDownloadEvent) => void; /** Per-file metadata for a file inside a lazy archive. */ export interface LazyArchiveFileEntry { ino: number; + /** Inode-slot generation; omitted only by legacy serialized metadata. */ + generation?: number; + /** Inode data-mutation sequence; omitted only by legacy metadata. */ + dataSequence?: number; size: number; isSymlink: boolean; deleted: boolean; + /** True once this inode's archive backing is no longer pending. */ + materialized?: boolean; + /** Original path inside the archive (stable across VFS rename/hard-link). */ + archivePath?: string; } /** @@ -61,9 +77,13 @@ export interface SerializedLazyArchiveEntry { entries: Array<{ vfsPath: string; ino: number; + generation?: number; + dataSequence?: number; size: number; isSymlink: boolean; deleted: boolean; + materialized?: boolean; + archivePath?: string; }>; } @@ -118,7 +138,9 @@ const COPY_CHUNK_BYTES = 1024 * 1024; const MIN_REBASE_INITIAL_BYTES = 16 * 1024 * 1024; const VFS_IMAGE_MAX_METADATA_BYTES = 64 * 1024; -function cloneMetadata(metadata: VfsImageMetadata | null): VfsImageMetadata | null { +function cloneMetadata( + metadata: VfsImageMetadata | null, +): VfsImageMetadata | null { return metadata === null ? null : { ...metadata }; } @@ -127,15 +149,22 @@ function validateMetadata(metadata: VfsImageMetadata): VfsImageMetadata { throw new Error("VFS image metadata must be an object"); } if (metadata.version !== 1) { - throw new Error(`Unsupported VFS image metadata version: ${String(metadata.version)}`); + throw new Error( + `Unsupported VFS image metadata version: ${String(metadata.version)}`, + ); } if ( metadata.kernelAbi !== undefined && (!Number.isInteger(metadata.kernelAbi) || metadata.kernelAbi < 0) ) { - throw new Error(`VFS image metadata kernelAbi must be a non-negative integer`); + throw new Error( + `VFS image metadata kernelAbi must be a non-negative integer`, + ); } - if (metadata.createdBy !== undefined && typeof metadata.createdBy !== "string") { + if ( + metadata.createdBy !== undefined && + typeof metadata.createdBy !== "string" + ) { throw new Error("VFS image metadata createdBy must be a string"); } return { ...metadata }; @@ -196,11 +225,7 @@ function parseImageHeader(input: Uint8Array): ParsedImageHeader { throw new Error("VFS image too small"); } - const view = new DataView( - image.buffer, - image.byteOffset, - image.byteLength, - ); + const view = new DataView(image.buffer, image.byteOffset, image.byteLength); const magic = view.getUint32(0, true); if (magic !== VFS_IMAGE_MAGIC) { throw new Error( @@ -270,12 +295,23 @@ function concatChunks(chunks: Uint8Array[], total: number): Uint8Array { export class MemoryFileSystem implements FileSystemBackend { private fs: SharedFS; private imageMetadata: VfsImageMetadata | null; - /** Lazy files: inode → { path, url, size }. Cleared per-inode after materialization. */ - private lazyFiles = new Map(); + /** Lazy files keyed by inode slot + generation (raw inode numbers are reused). */ + private lazyFiles = new Map< + string, + { + ino: number; + generation: number; + dataSequence: number; + path: string; + paths: Set; + url: string; + size: number; + } + >(); /** Lazy archive groups (bundle of files backed by one zip URL). */ private lazyArchiveGroups: LazyArchiveGroup[] = []; - /** Fast lookup: inode → group it belongs to. Cleared per-group after materialization. */ - private lazyArchiveInodes = new Map(); + /** Fast lookup keyed by inode slot + generation. */ + private lazyArchiveInodes = new Map(); private lazyDownloadListeners = new Set(); private constructor(fs: SharedFS, metadata: VfsImageMetadata | null = null) { @@ -283,12 +319,177 @@ export class MemoryFileSystem implements FileSystemBackend { this.imageMetadata = metadata; } + private static inodeKey(ino: number, generation: number): string { + return `${ino}:${generation}`; + } + + private static canAdoptLegacyLazyStub(st: SfsStatResult): boolean { + // Images from before data-sequence tracking stored regular lazy entries as + // untouched zero-length stubs. Current registration performs one initial + // O_TRUNC, so any later mutation sequence (or concrete bytes) is unsafe to + // associate with metadata that cannot name the content version it saw. + return ( + (st.mode & S_IFMT) === S_IFREG && st.size === 0 && st.dataSequence <= 1 + ); + } + + /** + * Reconcile process-local lazy metadata with authoritative SharedFS names. + * The identity map may come from the same transaction as a filesystem + * snapshot, so callers can serialize matching bytes and lazy paths. + */ + private reconcileLazyIdentityState( + identities: Map, + ): void { + for (const [key, entry] of this.lazyFiles) { + const identity = identities.get(key); + if ( + !identity || + identity.dataSequence !== entry.dataSequence || + identity.paths.length === 0 + ) { + this.lazyFiles.delete(key); + continue; + } + entry.paths = new Set(identity.paths); + if (!entry.paths.has(entry.path)) { + entry.path = identity.paths[0]; + } + } + + this.lazyArchiveInodes.clear(); + for (const group of this.lazyArchiveGroups) { + const pendingByIdentity = new Map(); + for (const entry of group.entries.values()) { + if ( + entry.deleted || + entry.materialized || + entry.generation === undefined + ) + continue; + const key = MemoryFileSystem.inodeKey(entry.ino, entry.generation); + if (!pendingByIdentity.has(key)) pendingByIdentity.set(key, entry); + } + + const reconciled = new Map(); + for (const [key, entry] of pendingByIdentity) { + const identity = identities.get(key); + if (!identity || identity.dataSequence !== (entry.dataSequence ?? 0)) + continue; + for (const path of identity.paths) { + reconciled.set(path, { + ...entry, + ino: identity.ino, + generation: identity.generation, + dataSequence: identity.dataSequence, + deleted: false, + materialized: false, + }); + } + if (identity.paths.length > 0) { + this.lazyArchiveInodes.set(key, group); + } + } + group.entries = reconciled; + group.materialized = reconciled.size === 0; + } + } + + private lazyFileForStat(st: SfsStatResult) { + const key = MemoryFileSystem.inodeKey(st.ino, st.generation); + const entry = this.lazyFiles.get(key); + if (entry && entry.dataSequence !== st.dataSequence) { + this.lazyFiles.delete(key); + return undefined; + } + return entry; + } + + private lazyArchiveForStat(st: SfsStatResult) { + const key = MemoryFileSystem.inodeKey(st.ino, st.generation); + const group = this.lazyArchiveInodes.get(key); + if (!group) return undefined; + const entries = Array.from(group.entries.values()).filter( + (entry) => + entry.ino === st.ino && + entry.generation === st.generation && + !entry.deleted && + !entry.materialized, + ); + if (entries.some((entry) => entry.dataSequence === st.dataSequence)) { + return group; + } + this.lazyArchiveInodes.delete(key); + for (const entry of entries) entry.materialized = true; + return undefined; + } + + /** A successful guest data mutation makes any deferred backing obsolete. */ + private invalidateLazyData(st: SfsStatResult): void { + const key = MemoryFileSystem.inodeKey(st.ino, st.generation); + this.lazyFiles.delete(key); + + const group = this.lazyArchiveInodes.get(key); + if (!group) return; + this.lazyArchiveInodes.delete(key); + for (const entry of group.entries.values()) { + if (entry.ino === st.ino && entry.generation === st.generation) { + // Keep the concrete inode in the image, but prevent a later archive + // fetch from overwriting data the guest supplied through any alias. + entry.materialized = true; + } + } + } + + private rewriteLazyNamespacePaths( + source: NamespaceEntryIdentity, + oldPath: string, + newPath: string, + ): void { + const oldBase = oldPath.length > 1 ? oldPath.replace(/\/+$/, "") : oldPath; + const newBase = newPath.length > 1 ? newPath.replace(/\/+$/, "") : newPath; + const oldPrefix = `${oldBase}/`; + const newPrefix = `${newBase}/`; + const sourceKey = MemoryFileSystem.inodeKey(source.ino, source.generation); + const directory = (source.mode & S_IFMT) === S_IFDIR; + const rewrite = (candidate: string): string => + candidate === oldBase + ? newBase + : directory && candidate.startsWith(oldPrefix) + ? newPrefix + candidate.slice(oldPrefix.length) + : candidate; + + for (const [key, lazy] of this.lazyFiles) { + if (!directory && key !== sourceKey) continue; + lazy.paths = new Set(Array.from(lazy.paths, rewrite)); + lazy.path = rewrite(lazy.path); + } + + for (const group of this.lazyArchiveGroups) { + const rewritten = new Map(); + for (const [candidate, entry] of group.entries) { + const entryKey = + entry.generation === undefined + ? null + : MemoryFileSystem.inodeKey(entry.ino, entry.generation); + rewritten.set( + directory || entryKey === sourceKey ? rewrite(candidate) : candidate, + entry, + ); + } + group.entries = rewritten; + } + } + /** Return the underlying SharedArrayBuffer (for sharing with workers). */ get sharedBuffer(): SharedArrayBuffer { return this.fs.buffer as SharedArrayBuffer; } - static create(sab: SharedArrayBuffer, maxSizeBytes?: number): MemoryFileSystem { + static create( + sab: SharedArrayBuffer, + maxSizeBytes?: number, + ): MemoryFileSystem { return new MemoryFileSystem(SharedFS.mkfs(sab, maxSizeBytes)); } @@ -303,24 +504,43 @@ export class MemoryFileSystem implements FileSystemBackend { */ rebaseToNewFileSystem(maxByteLength: number): MemoryFileSystem { if (!Number.isSafeInteger(maxByteLength) || maxByteLength <= 0) { - throw new Error(`Invalid MemoryFileSystem maxByteLength: ${maxByteLength}`); + throw new Error( + `Invalid MemoryFileSystem maxByteLength: ${maxByteLength}`, + ); } - const initialByteLength = Math.min( - maxByteLength, - Math.max(this.sharedBuffer.byteLength, MIN_REBASE_INITIAL_BYTES), - ); const SharedArrayBufferCtor = SharedArrayBuffer as new ( byteLength: number, options?: { maxByteLength?: number }, ) => SharedArrayBuffer; + + // Copy from one quiescent source image. Exporting lazy paths and then + // walking the live SAB would let a peer rename an entry between those two + // operations, making the logical lazy size disagree with the copied path. + const { bytes: sourceBytes, identities } = this.fs.snapshotState(); + this.reconcileLazyIdentityState(identities); + const lazyEntries = this.serializeLazyEntries(); + const lazyArchiveEntries = this.serializeLazyArchiveEntries(); + const sourceSab = new SharedArrayBufferCtor(sourceBytes.byteLength); + new Uint8Array(sourceSab).set(sourceBytes); + const source = new MemoryFileSystem( + SharedFS.mount(sourceSab, { restoreImage: true }), + this.imageMetadata, + ); + source.importLazyEntries(lazyEntries); + source.importLazyArchiveEntries(lazyArchiveEntries); + + const initialByteLength = Math.min( + maxByteLength, + Math.max(sourceBytes.byteLength, MIN_REBASE_INITIAL_BYTES), + ); const sab = new SharedArrayBufferCtor(initialByteLength, { maxByteLength }); const target = MemoryFileSystem.create(sab, maxByteLength); target.setImageMetadata(this.imageMetadata); - const lazyEntries = this.exportLazyEntries(); - const lazyFilePaths = new Set(lazyEntries.map((entry) => entry.path)); - const lazyArchiveEntries = this.exportLazyArchiveEntries(); + const lazyFilePaths = new Set( + lazyEntries.flatMap((entry) => entry.paths ?? [entry.path]), + ); const lazyArchiveStubPaths = new Set(); for (const group of lazyArchiveEntries) { if (group.materialized) continue; @@ -331,19 +551,40 @@ export class MemoryFileSystem implements FileSystemBackend { } } - this.copyPathToFreshFileSystem("/", target, lazyFilePaths, lazyArchiveStubPaths); - - target.importLazyEntries(lazyEntries.map((entry) => ({ - ...entry, - ino: target.lstat(entry.path).ino, - }))); - target.importLazyArchiveEntries(lazyArchiveEntries.map((group) => ({ - ...group, - entries: group.entries.map((entry) => ({ - ...entry, - ino: entry.deleted ? 0 : target.lstat(entry.vfsPath).ino, + source.copyPathToFreshFileSystem( + "/", + target, + lazyFilePaths, + lazyArchiveStubPaths, + new Map(), + ); + + target.importLazyEntries( + lazyEntries.map((entry) => { + const st = target.fs.lstat(entry.path); + return { + ...entry, + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + }; + }), + ); + target.importLazyArchiveEntries( + lazyArchiveEntries.map((group) => ({ + ...group, + entries: group.entries.map((entry) => { + if (entry.deleted) return { ...entry, ino: 0, generation: undefined }; + const st = target.fs.lstat(entry.vfsPath); + return { + ...entry, + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + }; + }), })), - }))); + ); return target; } @@ -367,20 +608,22 @@ export class MemoryFileSystem implements FileSystemBackend { if (this.lazyDownloadListeners.size === 0) return; const stamped: LazyDownloadEvent = { ...event, t: monotonicNow() }; for (const listener of this.lazyDownloadListeners) { - try { listener(stamped); } catch { /* listener errors must not break VFS I/O */ } + try { + listener(stamped); + } catch { + /* listener errors must not break VFS I/O */ + } } } - private async fetchLazyBytes( - details: { - id: string; - kind: LazyDownloadKind; - url: string; - path?: string; - mountPrefix?: string; - fallbackTotalBytes?: number; - }, - ): Promise { + private async fetchLazyBytes(details: { + id: string; + kind: LazyDownloadKind; + url: string; + path?: string; + mountPrefix?: string; + fallbackTotalBytes?: number; + }): Promise { let loadedBytes = 0; let totalBytes = details.fallbackTotalBytes; const base = { @@ -466,23 +709,38 @@ export class MemoryFileSystem implements FileSystemBackend { /** * Register a lazy file: creates an empty stub in SharedFS and records - * metadata so that read() will fetch content on demand via sync XHR. + * metadata for ensureMaterialized() to fetch asynchronously before a + * synchronous read or exec path consumes the file. * Returns the inode number (useful for forwarding to other instances). */ - registerLazyFile(path: string, url: string, size: number, mode = 0o755): number { + registerLazyFile( + path: string, + url: string, + size: number, + mode = 0o755, + ): number { // Ensure parent directories exist const parts = path.split("/").filter(Boolean); let current = ""; for (let i = 0; i < parts.length - 1; i++) { current += "/" + parts[i]; - try { this.fs.mkdir(current, 0o755); } catch { /* exists */ } + try { + this.fs.mkdir(current, 0o755); + } catch { + /* exists */ + } } - // Create empty stub file - const fd = this.fs.open(path, 0o1101, mode); // O_WRONLY | O_CREAT | O_TRUNC - this.fs.close(fd); - // Get inode - const st = this.fs.stat(path); - this.lazyFiles.set(st.ino, { path, url, size }); + const st = this.fs.createLazyStub(path, mode); + this.invalidateLazyData(st); + this.lazyFiles.set(MemoryFileSystem.inodeKey(st.ino, st.generation), { + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + path, + paths: new Set([path]), + url, + size, + }); return st.ino; } @@ -491,26 +749,105 @@ export class MemoryFileSystem implements FileSystemBackend { * Does not create files — assumes the files already exist in the SharedArrayBuffer. */ importLazyEntries(entries: LazyFileEntry[]): void { + this.importLazyEntriesInternal(entries, false); + } + + private importLazyEntriesInternal( + entries: LazyFileEntry[], + trustedLegacySnapshot: boolean, + ): void { for (const e of entries) { - this.lazyFiles.set(e.ino, { path: e.path, url: e.url, size: e.size }); + const isLegacy = + e.generation === undefined || e.dataSequence === undefined; + if (isLegacy && !trustedLegacySnapshot) { + throw new Error( + "Live lazy-file metadata requires inode generation and data sequence", + ); + } + const validPaths = new Set(); + let identity: SfsStatResult | null = null; + for (const path of new Set([e.path, ...(e.paths ?? [])])) { + let st: SfsStatResult; + try { + st = this.fs.stat(path); + } catch { + continue; + } + if (st.ino !== e.ino) continue; + if (e.generation !== undefined && st.generation !== e.generation) { + continue; + } + if (e.dataSequence === undefined) { + if (!MemoryFileSystem.canAdoptLegacyLazyStub(st)) continue; + } else if (st.dataSequence !== e.dataSequence) continue; + identity ??= st; + validPaths.add(path); + } + if (!identity || validPaths.size === 0) continue; + const primaryPath = validPaths.has(e.path) + ? e.path + : validPaths.values().next().value!; + this.lazyFiles.set( + MemoryFileSystem.inodeKey(identity.ino, identity.generation), + { + ino: identity.ino, + generation: identity.generation, + dataSequence: identity.dataSequence, + path: primaryPath, + paths: validPaths, + url: e.url, + size: e.size, + }, + ); } } - /** Export all pending lazy entries for transfer to another instance. */ - exportLazyEntries(): LazyFileEntry[] { + private serializeLazyEntries(): LazyFileEntry[] { const entries: LazyFileEntry[] = []; - for (const [ino, { path, url, size }] of this.lazyFiles) { - entries.push({ ino, path, url, size }); + for (const { + ino, + generation, + dataSequence, + path, + paths, + url, + size, + } of this.lazyFiles.values()) { + entries.push({ + ino, + generation, + dataSequence, + path, + paths: Array.from(paths), + url, + size, + }); } return entries; } + /** Export all pending lazy entries for transfer to another instance. */ + exportLazyEntries(): LazyFileEntry[] { + this.reconcileLazyIdentityState(this.fs.identityState()); + return this.serializeLazyEntries(); + } + /** Return lazy metadata for `path`, following symlinks through stat(). */ getLazyEntry(path: string): LazyFileEntry | null { try { const st = this.fs.stat(path); - const entry = this.lazyFiles.get(st.ino); - return entry ? { ino: st.ino, path: entry.path, url: entry.url, size: entry.size } : null; + const entry = this.lazyFileForStat(st); + return entry + ? { + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + path: entry.path, + paths: Array.from(entry.paths), + url: entry.url, + size: entry.size, + } + : null; } catch { return null; } @@ -522,11 +859,8 @@ export class MemoryFileSystem implements FileSystemBackend { * them with bundler-produced asset URLs. */ rewriteLazyFileUrls(transform: (url: string, path: string) => string): void { - for (const [ino, entry] of this.lazyFiles) { - this.lazyFiles.set(ino, { - ...entry, - url: transform(entry.url, entry.path), - }); + for (const entry of this.lazyFiles.values()) { + entry.url = transform(entry.url, entry.path); } } @@ -562,55 +896,132 @@ export class MemoryFileSystem implements FileSystemBackend { let current = ""; for (let i = 0; i < parts.length - 1; i++) { current += "/" + parts[i]; - try { this.fs.mkdir(current, 0o755); } catch { /* exists */ } + try { + this.fs.mkdir(current, 0o755); + } catch { + /* exists */ + } } - if (ze.isSymlink && symlinkTargets?.has(ze.fileName)) { + if (ze.isSymlink) { + if (!symlinkTargets?.has(ze.fileName)) { + throw new Error( + `Lazy archive symlink target was not provided: ${ze.fileName}`, + ); + } const target = symlinkTargets.get(ze.fileName)!; this.fs.symlink(target, vfsPath); + const st = this.fs.lstat(vfsPath); + const entry: LazyArchiveFileEntry = { + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + size: ze.uncompressedSize, + isSymlink: true, + deleted: false, + materialized: true, + archivePath: ze.fileName, + }; + group.entries.set(vfsPath, entry); } else { - const fd = this.fs.open(vfsPath, 0o1101, ze.mode); // O_WRONLY | O_CREAT | O_TRUNC - this.fs.close(fd); + const st = this.fs.createLazyStub(vfsPath, ze.mode); + this.invalidateLazyData(st); + const entry: LazyArchiveFileEntry = { + ino: st.ino, + generation: st.generation, + dataSequence: st.dataSequence, + size: ze.uncompressedSize, + isSymlink: false, + deleted: false, + materialized: false, + archivePath: ze.fileName, + }; + group.entries.set(vfsPath, entry); + this.lazyArchiveInodes.set( + MemoryFileSystem.inodeKey(st.ino, st.generation), + group, + ); } - - const st = this.fs.lstat(vfsPath); - const entry: LazyArchiveFileEntry = { - ino: st.ino, - size: ze.uncompressedSize, - isSymlink: ze.isSymlink, - deleted: false, - }; - group.entries.set(vfsPath, entry); - this.lazyArchiveInodes.set(st.ino, group); } + group.materialized = Array.from(group.entries.values()).every( + (entry) => entry.deleted || entry.materialized, + ); this.lazyArchiveGroups.push(group); return group; } /** Import lazy archive groups from another instance. Assumes stubs already exist. */ importLazyArchiveEntries(serialized: SerializedLazyArchiveEntry[]): void { + this.importLazyArchiveEntriesInternal(serialized, false); + } + + private importLazyArchiveEntriesInternal( + serialized: SerializedLazyArchiveEntry[], + trustedLegacySnapshot: boolean, + ): void { for (const s of serialized) { const entries = new Map(); + const normalizedPrefix = s.mountPrefix.replace(/\/+$/, ""); for (const e of s.entries) { + let st: SfsStatResult | null = null; + const materialized = + s.materialized || e.materialized === true || e.isSymlink; + if (!e.deleted && !materialized) { + const isLegacy = + e.generation === undefined || e.dataSequence === undefined; + if (isLegacy && !trustedLegacySnapshot) { + throw new Error( + "Live lazy-archive metadata requires inode generation and data sequence", + ); + } + try { + st = this.fs.lstat(e.vfsPath); + } catch { + continue; + } + if (st.ino !== e.ino) continue; + if (e.generation !== undefined && st.generation !== e.generation) { + continue; + } + if (e.dataSequence === undefined) { + if (!MemoryFileSystem.canAdoptLegacyLazyStub(st)) continue; + } else if (st.dataSequence !== e.dataSequence) continue; + } entries.set(e.vfsPath, { ino: e.ino, + generation: st?.generation ?? e.generation, + dataSequence: st?.dataSequence ?? e.dataSequence, size: e.size, isSymlink: e.isSymlink, deleted: e.deleted, + materialized, + archivePath: + e.archivePath ?? e.vfsPath.slice(normalizedPrefix.length + 1), }); } const group: LazyArchiveGroup = { url: s.url, mountPrefix: s.mountPrefix, - materialized: s.materialized, + materialized: + s.materialized || + Array.from(entries.values()).every( + (entry) => entry.deleted || entry.materialized, + ), entries, }; this.lazyArchiveGroups.push(group); if (!group.materialized) { for (const [, entry] of entries) { - if (!entry.deleted) { - this.lazyArchiveInodes.set(entry.ino, group); + if ( + !entry.deleted && + !entry.materialized && + entry.generation !== undefined + ) { + this.lazyArchiveInodes.set( + MemoryFileSystem.inodeKey(entry.ino, entry.generation), + group, + ); } } } @@ -628,20 +1039,35 @@ export class MemoryFileSystem implements FileSystemBackend { } } - /** Export all lazy archive groups for transfer to another instance. */ - exportLazyArchiveEntries(): SerializedLazyArchiveEntry[] { - return this.lazyArchiveGroups.map((group) => ({ - url: group.url, - mountPrefix: group.mountPrefix, - materialized: group.materialized, - entries: Array.from(group.entries, ([vfsPath, entry]) => ({ + private serializeLazyArchiveEntries(): SerializedLazyArchiveEntry[] { + const serialized: SerializedLazyArchiveEntry[] = []; + for (const group of this.lazyArchiveGroups) { + const entries = Array.from(group.entries, ([vfsPath, entry]) => ({ vfsPath, ino: entry.ino, + generation: entry.generation, + dataSequence: entry.dataSequence, size: entry.size, isSymlink: entry.isSymlink, deleted: entry.deleted, - })), - })); + materialized: entry.materialized, + archivePath: entry.archivePath, + })).filter((entry) => !entry.deleted && !entry.materialized); + if (entries.length === 0) continue; + serialized.push({ + url: group.url, + mountPrefix: group.mountPrefix, + materialized: false, + entries, + }); + } + return serialized; + } + + /** Export all pending lazy archive groups for transfer to another instance. */ + exportLazyArchiveEntries(): SerializedLazyArchiveEntry[] { + this.reconcileLazyIdentityState(this.fs.identityState()); + return this.serializeLazyArchiveEntries(); } /** @@ -651,33 +1077,58 @@ export class MemoryFileSystem implements FileSystemBackend { * Returns true if something was materialized, false if already concrete. */ async ensureMaterialized(path: string): Promise { - if (this.lazyFiles.size === 0 && this.lazyArchiveInodes.size === 0) return false; - try { - const st = this.fs.stat(path); // follows symlinks - const entry = this.lazyFiles.get(st.ino); - if (entry) { - const data = await this.fetchLazyBytes({ - id: `file:${st.ino}`, - kind: "file", - url: entry.url, - path: entry.path, - fallbackTotalBytes: entry.size, - }); - const fd = this.fs.open(entry.path, 0o1101, 0o755); // O_WRONLY | O_CREAT | O_TRUNC - this.fs.write(fd, data); - this.fs.close(fd); - this.lazyFiles.delete(st.ino); - return true; - } - const group = this.lazyArchiveInodes.get(st.ino); - if (group) { - await this.ensureArchiveMaterialized(group); - return true; - } + if (this.lazyFiles.size === 0 && this.lazyArchiveInodes.size === 0) return false; + let st: SfsStatResult; + try { + st = this.fs.stat(path); // follows symlinks } catch { return false; } + const key = MemoryFileSystem.inodeKey(st.ino, st.generation); + const entry = this.lazyFiles.get(key); + if (entry) { + const data = await this.fetchLazyBytes({ + id: `file:${st.ino}`, + kind: "file", + url: entry.url, + path: entry.path, + fallbackTotalBytes: entry.size, + }); + for (let attempt = 0; attempt < 3; attempt++) { + if (this.lazyFiles.get(key) !== entry) return false; + for (const candidate of new Set([path, ...entry.paths])) { + const materialized = this.fs.replaceIfIdentity( + candidate, + entry.ino, + entry.generation, + entry.dataSequence, + data, + ); + if (materialized) { + entry.path = candidate; + this.lazyFiles.delete(key); + return true; + } + } + // A peer may have renamed the inode while the fetch was in flight. + // Refresh aliases and retry immediately with the bytes already read. + this.reconcileLazyIdentityState(this.fs.identityState()); + } + throw new Error( + `Lazy file kept changing names while materializing: ${path}`, + ); + } + const group = this.lazyArchiveInodes.get(key); + if (group) { + await this.ensureArchiveMaterialized(group, { + path, + ino: st.ino, + generation: st.generation, + }); + return !this.lazyArchiveInodes.has(key); + } + return false; } /** @@ -685,7 +1136,10 @@ export class MemoryFileSystem implements FileSystemBackend { * central directory, and write every non-deleted entry into its stub. * Subsequent calls are no-ops. */ - async ensureArchiveMaterialized(group: LazyArchiveGroup): Promise { + async ensureArchiveMaterialized( + group: LazyArchiveGroup, + requested?: { path: string; ino: number; generation: number }, + ): Promise { if (group.materialized) return; const zipData = await this.fetchLazyBytes({ @@ -701,21 +1155,98 @@ export class MemoryFileSystem implements FileSystemBackend { for (const ze of zipEntries) zipLookup.set(ze.fileName, ze); const normalizedPrefix = group.mountPrefix.replace(/\/+$/, ""); - for (const [vfsPath, archiveEntry] of group.entries) { - if (archiveEntry.deleted) continue; - if (archiveEntry.isSymlink) continue; // symlinks already created at registration - const zipFileName = vfsPath.slice(normalizedPrefix.length + 1); - const ze = zipLookup.get(zipFileName); - if (!ze) continue; - const content = extractZipEntry(zipData, ze); - const fd = this.fs.open(vfsPath, 0o1101, 0o755); // O_WRONLY | O_CREAT | O_TRUNC - if (content.length > 0) this.fs.write(fd, content); - this.fs.close(fd); + const requestedKey = requested + ? MemoryFileSystem.inodeKey(requested.ino, requested.generation) + : null; + for (let attempt = 0; attempt < 3; attempt++) { + for (const [vfsPath, archiveEntry] of group.entries) { + if (archiveEntry.deleted || archiveEntry.materialized) continue; + if (archiveEntry.isSymlink) { + archiveEntry.materialized = true; + continue; + } + const zipFileName = + archiveEntry.archivePath ?? + vfsPath.slice(normalizedPrefix.length + 1); + const ze = zipLookup.get(zipFileName); + if (!ze) continue; + const content = extractZipEntry(zipData, ze); + if (archiveEntry.generation === undefined) continue; + const key = MemoryFileSystem.inodeKey( + archiveEntry.ino, + archiveEntry.generation, + ); + if (this.lazyArchiveInodes.get(key) !== group) continue; + const candidates = new Set([vfsPath]); + if ( + requested && + requested.ino === archiveEntry.ino && + requested.generation === archiveEntry.generation + ) + candidates.add(requested.path); + let materialized = false; + for (const candidate of candidates) { + materialized = this.fs.replaceIfIdentity( + candidate, + archiveEntry.ino, + archiveEntry.generation, + archiveEntry.dataSequence ?? 0, + content, + ); + if (materialized) break; + } + if (!materialized) continue; + this.lazyArchiveInodes.delete(key); + for (const alias of group.entries.values()) { + if ( + alias.ino === archiveEntry.ino && + alias.generation === archiveEntry.generation + ) + alias.materialized = true; + } + } + + group.materialized = Array.from(group.entries.values()).every( + (entry) => entry.deleted || entry.materialized, + ); + if (group.materialized) return; + this.reconcileLazyIdentityState(this.fs.identityState()); + if (requestedKey && !this.lazyArchiveInodes.has(requestedKey)) return; } - group.materialized = true; - for (const [, archiveEntry] of group.entries) { - this.lazyArchiveInodes.delete(archiveEntry.ino); + if (requestedKey && this.lazyArchiveInodes.has(requestedKey)) { + throw new Error( + `Lazy archive member kept changing names while materializing: ${requested?.path}`, + ); + } + } + + private async materializeAllLazyEntries(): Promise { + // A peer can rename an inode while an asynchronous fetch is in flight. + // Refresh and retry a bounded number of times; a continuously mutating + // filesystem is not a stable source for a self-contained image. + for (let attempt = 0; attempt < 3; attempt++) { + this.reconcileLazyIdentityState(this.fs.identityState()); + if (this.lazyFiles.size === 0 && this.lazyArchiveInodes.size === 0) + return; + + const filePaths = Array.from( + this.lazyFiles.values(), + (entry) => entry.path, + ); + for (const path of filePaths) await this.ensureMaterialized(path); + + const archiveGroups = new Set(this.lazyArchiveInodes.values()); + for (const group of archiveGroups) { + await this.ensureArchiveMaterialized(group); + } + } + + this.reconcileLazyIdentityState(this.fs.identityState()); + if (this.lazyFiles.size !== 0 || this.lazyArchiveInodes.size !== 0) { + throw new Error( + "Cannot create a self-contained VFS image while lazy entries remain pending", + ); } } @@ -729,28 +1260,25 @@ export class MemoryFileSystem implements FileSystemBackend { */ async saveImage(options?: VfsImageOptions): Promise { if (options?.materializeAll) { - const paths = Array.from(this.lazyFiles.values()).map((e) => e.path); - for (const p of paths) { - await this.ensureMaterialized(p); - } + await this.materializeAllLazyEntries(); } - const sabBytes = new Uint8Array(this.fs.buffer); - const lazyEntries = this.exportLazyEntries(); + const { bytes: sabBytes, identities } = this.fs.snapshotState(); + this.reconcileLazyIdentityState(identities); + const lazyEntries = this.serializeLazyEntries(); const hasLazy = lazyEntries.length > 0; const lazyJson = hasLazy ? new TextEncoder().encode(JSON.stringify(lazyEntries)) : new Uint8Array(0); - const archiveEntries = this.exportLazyArchiveEntries(); + const archiveEntries = this.serializeLazyArchiveEntries(); const hasArchives = archiveEntries.length > 0; const archiveJson = hasArchives ? new TextEncoder().encode(JSON.stringify(archiveEntries)) : new Uint8Array(0); - const metadata = options?.metadata === undefined - ? this.imageMetadata - : options.metadata; + const metadata = + options?.metadata === undefined ? this.imageMetadata : options.metadata; const metadataJson = encodeMetadata(metadata); const hasMetadata = metadataJson.byteLength > 0; @@ -760,11 +1288,11 @@ export class MemoryFileSystem implements FileSystemBackend { const metadataSectionSize = hasMetadata ? 4 + metadataJson.byteLength : 0; const totalSize = VFS_IMAGE_HEADER_SIZE + - sabBytes.byteLength + - 4 + - lazyJson.byteLength + - archiveSectionSize + - metadataSectionSize; + sabBytes.byteLength + + 4 + + lazyJson.byteLength + + archiveSectionSize + + metadataSectionSize; const image = new Uint8Array(totalSize); const view = new DataView(image.buffer); @@ -780,10 +1308,8 @@ export class MemoryFileSystem implements FileSystemBackend { ); view.setUint32(12, sabBytes.byteLength, true); - // SAB data — copy from SharedArrayBuffer (can't use set() directly on SAB-backed views in all environments) - const sabCopy = new Uint8Array(sabBytes.byteLength); - sabCopy.set(sabBytes); - image.set(sabCopy, VFS_IMAGE_HEADER_SIZE); + // SAB data is already a detached, runtime-state-free snapshot. + image.set(sabBytes, VFS_IMAGE_HEADER_SIZE); // Lazy entries const lazyOffset = VFS_IMAGE_HEADER_SIZE + sabBytes.byteLength; @@ -801,7 +1327,8 @@ export class MemoryFileSystem implements FileSystemBackend { // Metadata if (hasMetadata) { - const metadataOffset = lazyOffset + 4 + lazyJson.byteLength + archiveSectionSize; + const metadataOffset = + lazyOffset + 4 + lazyJson.byteLength + archiveSectionSize; view.setUint32(metadataOffset, metadataJson.byteLength, true); image.set(metadataJson, metadataOffset + 4); } @@ -833,7 +1360,10 @@ export class MemoryFileSystem implements FileSystemBackend { } if (metadataLen === 0) return null; return decodeMetadata( - parsed.image.subarray(metadataOffset + 4, metadataOffset + 4 + metadataLen), + parsed.image.subarray( + metadataOffset + 4, + metadataOffset + 4 + metadataLen, + ), ); } @@ -866,7 +1396,10 @@ export class MemoryFileSystem implements FileSystemBackend { * so the filesystem can expand beyond the image's original size, up to the * maximum already recorded in the image superblock. */ - static fromImage(image: Uint8Array, options?: { maxByteLength?: number }): MemoryFileSystem { + static fromImage( + image: Uint8Array, + options?: { maxByteLength?: number }, + ): MemoryFileSystem { const parsed = parseImageHeader(image); image = parsed.image; const view = parsed.view; @@ -885,25 +1418,33 @@ export class MemoryFileSystem implements FileSystemBackend { ) => SharedArrayBuffer; const sab = new SharedArrayBufferCtor(sabLen, sabOptions); const sabView = new Uint8Array(sab); - sabView.set(image.subarray(VFS_IMAGE_HEADER_SIZE, VFS_IMAGE_HEADER_SIZE + sabLen)); + sabView.set( + image.subarray(VFS_IMAGE_HEADER_SIZE, VFS_IMAGE_HEADER_SIZE + sabLen), + ); let metadata: VfsImageMetadata | null = null; if (flags & VFS_IMAGE_FLAG_HAS_METADATA) { metadata = MemoryFileSystem.readImageMetadata(image); } - const mfs = new MemoryFileSystem(SharedFS.mount(sab), metadata); + const mfs = new MemoryFileSystem( + SharedFS.mount(sab, { restoreImage: true }), + metadata, + ); // Restore lazy entries const lazyOffset = VFS_IMAGE_HEADER_SIZE + sabLen; const lazyLen = view.getUint32(lazyOffset, true); if (flags & VFS_IMAGE_FLAG_HAS_LAZY) { if (lazyLen > 0) { - const lazyBytes = image.subarray(lazyOffset + 4, lazyOffset + 4 + lazyLen); + const lazyBytes = image.subarray( + lazyOffset + 4, + lazyOffset + 4 + lazyLen, + ); const entries: LazyFileEntry[] = JSON.parse( new TextDecoder().decode(lazyBytes), ); - mfs.importLazyEntries(entries); + mfs.importLazyEntriesInternal(entries, true); } } @@ -922,7 +1463,7 @@ export class MemoryFileSystem implements FileSystemBackend { const entries: SerializedLazyArchiveEntry[] = JSON.parse( new TextDecoder().decode(archiveBytes), ); - mfs.importLazyArchiveEntries(entries); + mfs.importLazyArchiveEntriesInternal(entries, true); } } @@ -944,8 +1485,37 @@ export class MemoryFileSystem implements FileSystemBackend { }; } + private adaptStatWithLazySize(s: SfsStatResult): StatResult { + const result = this.adaptStat(s); + const entry = this.lazyFileForStat(s); + if (entry) { + result.size = entry.size; + return result; + } + + const group = this.lazyArchiveForStat(s); + if (group) { + for (const archiveEntry of group.entries.values()) { + if ( + archiveEntry.ino === s.ino && + archiveEntry.generation === s.generation && + !archiveEntry.deleted + ) { + result.size = archiveEntry.size; + break; + } + } + } + return result; + } + open(path: string, flags: number, mode: number): number { - return this.fs.open(path, flags, mode); + const handle = this.fs.open(path, flags, mode); + if ((flags & 0x0200) !== 0) { + // O_TRUNC + this.invalidateLazyData(this.fs.fstat(handle)); + } + return handle; } close(handle: number): number { @@ -982,9 +1552,12 @@ export class MemoryFileSystem implements FileSystemBackend { this.fs.lseek(handle, offset, 0); // SEEK_SET const n = this.fs.write(handle, buffer.subarray(0, length)); this.fs.lseek(handle, savedPos, 0); // restore position + if (n > 0) this.invalidateLazyData(this.fs.fstat(handle)); return n; } - return this.fs.write(handle, buffer.subarray(0, length)); + const n = this.fs.write(handle, buffer.subarray(0, length)); + if (n > 0) this.invalidateLazyData(this.fs.fstat(handle)); + return n; } seek(handle: number, offset: number, whence: number): number { @@ -992,27 +1565,12 @@ export class MemoryFileSystem implements FileSystemBackend { } fstat(handle: number): StatResult { - const result = this.adaptStat(this.fs.fstat(handle)); - // Override size for unmaterialized lazy files / archive entries - const entry = this.lazyFiles.get(result.ino); - if (entry) { - result.size = entry.size; - } else { - const group = this.lazyArchiveInodes.get(result.ino); - if (group) { - for (const archiveEntry of group.entries.values()) { - if (archiveEntry.ino === result.ino) { - result.size = archiveEntry.size; - break; - } - } - } - } - return result; + return this.adaptStatWithLazySize(this.fs.fstat(handle)); } ftruncate(handle: number, length: number): void { this.fs.ftruncate(handle, length); + this.invalidateLazyData(this.fs.fstat(handle)); } // SharedFS is memory-backed, fsync is a no-op @@ -1026,43 +1584,11 @@ export class MemoryFileSystem implements FileSystemBackend { } stat(path: string): StatResult { - const result = this.adaptStat(this.fs.stat(path)); - // Override size for unmaterialized lazy files / archive entries - const entry = this.lazyFiles.get(result.ino); - if (entry) { - result.size = entry.size; - } else { - const group = this.lazyArchiveInodes.get(result.ino); - if (group) { - for (const archiveEntry of group.entries.values()) { - if (archiveEntry.ino === result.ino) { - result.size = archiveEntry.size; - break; - } - } - } - } - return result; + return this.adaptStatWithLazySize(this.fs.stat(path)); } lstat(path: string): StatResult { - const result = this.adaptStat(this.fs.lstat(path)); - // Override size for unmaterialized lazy files / archive entries - const entry = this.lazyFiles.get(result.ino); - if (entry) { - result.size = entry.size; - } else { - const group = this.lazyArchiveInodes.get(result.ino); - if (group) { - for (const archiveEntry of group.entries.values()) { - if (archiveEntry.ino === result.ino) { - result.size = archiveEntry.size; - break; - } - } - } - } - return result; + return this.adaptStatWithLazySize(this.fs.lstat(path)); } statfs(path: string): StatfsResult { @@ -1092,28 +1618,119 @@ export class MemoryFileSystem implements FileSystemBackend { } unlink(path: string): void { - // If the path belongs to an unmaterialized archive group, mark the entry - // as deleted so materialization skips it. - if (this.lazyArchiveInodes.size > 0) { - try { - const st = this.fs.lstat(path); - const group = this.lazyArchiveInodes.get(st.ino); - if (group) { - const entry = group.entries.get(path); - if (entry) entry.deleted = true; - this.lazyArchiveInodes.delete(st.ino); + const removed = this.fs.unlink(path); + const key = MemoryFileSystem.inodeKey(removed.ino, removed.generation); + if ( + removed.linkCount > 1 && + (this.lazyFiles.has(key) || this.lazyArchiveInodes.has(key)) + ) { + // A peer may have added hard-link names this instance never observed. + // Rebuild aliases from SharedFS instead of treating an empty local path + // set as proof that the inode disappeared. + this.reconcileLazyIdentityState(this.fs.identityState()); + return; + } + + const lazy = this.lazyFiles.get(key); + if (lazy) { + lazy.paths.delete(path); + if (removed.linkCount <= 1) { + this.lazyFiles.delete(key); + } else if (lazy.path === path) { + lazy.path = lazy.paths.values().next().value!; + } + } + + const group = this.lazyArchiveInodes.get(key); + if (group) { + const entry = group.entries.get(path); + if (removed.linkCount <= 1) { + for (const candidate of group.entries.values()) { + if ( + candidate.ino === removed.ino && + candidate.generation === removed.generation + ) + candidate.deleted = true; } - } catch { /* not present — unlink will raise the real error */ } + this.lazyArchiveInodes.delete(key); + } else if (entry) { + group.entries.delete(path); + } } - this.fs.unlink(path); } rename(oldPath: string, newPath: string): void { - this.fs.rename(oldPath, newPath); + const { source, replaced } = this.fs.rename(oldPath, newPath); + + if ( + replaced && + replaced.ino === source.ino && + replaced.generation === source.generation + ) + return; + + let reconciledNamespace = false; + if (replaced) { + const replacedKey = MemoryFileSystem.inodeKey( + replaced.ino, + replaced.generation, + ); + if ( + replaced.linkCount > 1 && + (this.lazyFiles.has(replacedKey) || + this.lazyArchiveInodes.has(replacedKey)) + ) { + // The replaced inode survived through a hard link that may have been + // created by a peer. One authoritative reconciliation updates both + // that alias and the source paths changed by rename(). + this.reconcileLazyIdentityState(this.fs.identityState()); + reconciledNamespace = true; + } + + const replacedLazy = this.lazyFiles.get(replacedKey); + if (!reconciledNamespace && replacedLazy) { + replacedLazy.paths.delete(newPath); + if (replaced.linkCount <= 1) { + this.lazyFiles.delete(replacedKey); + } else if (replacedLazy.path === newPath) { + replacedLazy.path = replacedLazy.paths.values().next().value!; + } + } + const replacedGroup = this.lazyArchiveInodes.get(replacedKey); + if (!reconciledNamespace && replacedGroup) { + const entry = replacedGroup.entries.get(newPath); + if (replaced.linkCount <= 1) { + if (entry) entry.deleted = true; + this.lazyArchiveInodes.delete(replacedKey); + } else if (entry) { + replacedGroup.entries.delete(newPath); + } + } + } + + if (!reconciledNamespace) { + this.rewriteLazyNamespacePaths(source, oldPath, newPath); + } } link(existingPath: string, newPath: string): void { - this.fs.link(existingPath, newPath); + const sourceIdentity = this.fs.link(existingPath, newPath); + const key = MemoryFileSystem.inodeKey( + sourceIdentity.ino, + sourceIdentity.generation, + ); + const lazy = this.lazyFiles.get(key); + if (lazy) lazy.paths.add(newPath); + + const group = this.lazyArchiveInodes.get(key); + if (group) { + const source = Array.from(group.entries.values()).find( + (entry) => + entry.ino === sourceIdentity.ino && + entry.generation === sourceIdentity.generation, + ); + if (source) group.entries.set(newPath, { ...source }); + } } symlink(target: string, path: string): void { @@ -1154,7 +1771,12 @@ export class MemoryFileSystem implements FileSystemBackend { this.chmod(path, mode); } - symlinkWithOwner(target: string, path: string, uid: number, gid: number): void { + symlinkWithOwner( + target: string, + path: string, + uid: number, + gid: number, + ): void { this.symlink(target, path); this.lchown(path, uid, gid); } @@ -1164,6 +1786,7 @@ export class MemoryFileSystem implements FileSystemBackend { target: MemoryFileSystem, lazyFilePaths: Set, lazyArchiveStubPaths: Set, + hardLinks: Map, ): void { const st = this.lstat(path); const kind = st.mode & S_IFMT; @@ -1188,6 +1811,7 @@ export class MemoryFileSystem implements FileSystemBackend { target, lazyFilePaths, lazyArchiveStubPaths, + hardLinks, ); } } finally { @@ -1197,8 +1821,16 @@ export class MemoryFileSystem implements FileSystemBackend { return; } + const identity = st.nlink > 1 ? `${st.dev}:${st.ino}` : null; + const existingHardLink = identity ? hardLinks.get(identity) : undefined; + if (existingHardLink) { + target.link(existingHardLink, path); + return; + } + if (kind === S_IFLNK) { target.symlinkWithOwner(this.readlink(path), path, st.uid, st.gid); + if (identity) hardLinks.set(identity, path); return; } @@ -1206,14 +1838,17 @@ export class MemoryFileSystem implements FileSystemBackend { throw new Error(`Unsupported file type while rebasing VFS: ${path}`); } - const isLazyStub = lazyFilePaths.has(path) || lazyArchiveStubPaths.has(path); + const isLazyStub = + lazyFilePaths.has(path) || lazyArchiveStubPaths.has(path); if (isLazyStub) { target.createFileWithOwner(path, mode, st.uid, st.gid, new Uint8Array(0)); MemoryFileSystem.applyTimes(target, path, st); + if (identity) hardLinks.set(identity, path); return; } this.copyRegularFileToFreshFileSystem(path, target, st, mode); + if (identity) hardLinks.set(identity, path); } private copyRegularFileToFreshFileSystem( @@ -1226,7 +1861,9 @@ export class MemoryFileSystem implements FileSystemBackend { let outFd: number | null = null; try { outFd = target.open(path, O_WRONLY_CREAT_TRUNC, mode); - const chunk = new Uint8Array(Math.min(COPY_CHUNK_BYTES, Math.max(1, st.size))); + const chunk = new Uint8Array( + Math.min(COPY_CHUNK_BYTES, Math.max(1, st.size)), + ); let remaining = st.size; while (remaining > 0) { const wanted = Math.min(chunk.byteLength, remaining); @@ -1258,7 +1895,11 @@ export class MemoryFileSystem implements FileSystemBackend { MemoryFileSystem.applyTimes(target, path, st); } - private static applyTimes(fs: MemoryFileSystem, path: string, st: StatResult): void { + private static applyTimes( + fs: MemoryFileSystem, + path: string, + st: StatResult, + ): void { const atimeSec = Math.floor(st.atimeMs / 1000); const atimeNsec = Math.floor((st.atimeMs - atimeSec * 1000) * 1_000_000); const mtimeSec = Math.floor(st.mtimeMs / 1000); @@ -1271,7 +1912,13 @@ export class MemoryFileSystem implements FileSystemBackend { this.fs.stat(path); } - utimensat(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { + utimensat( + path: string, + atimeSec: number, + atimeNsec: number, + mtimeSec: number, + mtimeNsec: number, + ): void { this.fs.utimens(path, atimeSec, atimeNsec, mtimeSec, mtimeNsec); } @@ -1285,8 +1932,10 @@ export class MemoryFileSystem implements FileSystemBackend { // Determine d_type from mode const mode = entry.stat.mode; let dtype = 0; // DT_UNKNOWN - if ((mode & 0xf000) === 0x8000) dtype = 8; // DT_REG - else if ((mode & 0xf000) === 0x4000) dtype = 4; // DT_DIR + if ((mode & 0xf000) === 0x8000) + dtype = 8; // DT_REG + else if ((mode & 0xf000) === 0x4000) + dtype = 4; // DT_DIR else if ((mode & 0xf000) === 0xa000) dtype = 10; // DT_LNK return { name: entry.name, type: dtype, ino: entry.stat.ino }; } diff --git a/host/src/vfs/sharedfs-vendor.ts b/host/src/vfs/sharedfs-vendor.ts index 91cafad5be..47ed03713a 100644 --- a/host/src/vfs/sharedfs-vendor.ts +++ b/host/src/vfs/sharedfs-vendor.ts @@ -66,11 +66,13 @@ export const EPERM = -1; export const ENOENT = -2; export const EIO = -5; export const EBADF = -9; +export const EBUSY = -16; export const EEXIST = -17; export const ENOTDIR = -20; export const EISDIR = -21; export const EINVAL = -22; export const EMFILE = -24; +export const EFBIG = -27; export const ENOSPC = -28; export const ENAMETOOLONG = -36; export const ENOTEMPTY = -39; @@ -93,6 +95,7 @@ const SB_BLOCK_BITMAP_BLOCKS = 48; const SB_INODE_TABLE_BLOCKS = 52; const SB_GENERATION = 56; const SB_GLOBAL_LOCK = 60; +const SB_NAMESPACE_LOCK = 64; const SB_MAX_SIZE_BLOCKS = 68; const SB_GROW_CHUNK_BLOCKS = 72; @@ -111,7 +114,9 @@ const INO_UID = 96; // u32 const INO_GID = 100; // u32 const INO_GENERATION = 104; // uint64, incremented when an inode slot is allocated const INO_OPEN_COUNT = 112; // u32, open fd references -// 116-127 reserved for future fields (flags, xattrs, etc.) +const INO_DIR_SEQUENCE = 116; // u32, incremented after every directory mutation +const INO_DATA_SEQUENCE = 120; // u32, incremented after explicit data mutation +// 124-127 reserved for future fields (flags, xattrs, etc.) // FD entry layout const FD_INO = 4; @@ -122,12 +127,16 @@ const FD_IS_DIR = 20; // Lock bits const WRITER_BIT = 0x80000000 | 0; // -2147483648 as int32 const READER_MASK = 0x7fffffff | 0; +const MAX_FILE_BLOCKS = + DIRECT_BLOCKS + PTRS_PER_BLOCK + PTRS_PER_BLOCK * PTRS_PER_BLOCK; +const MAX_FILE_SIZE = MAX_FILE_BLOCKS * BLOCK_SIZE; // ── Types ──────────────────────────────────────────────────────────── export interface StatResult { ino: number; generation: number; + dataSequence: number; mode: number; linkCount: number; size: number; @@ -147,6 +156,25 @@ export interface SharedFsStats { maxName: number; } +export interface SharedFsIdentityState { + ino: number; + generation: number; + dataSequence: number; + paths: string[]; +} + +export interface NamespaceEntryIdentity { + ino: number; + generation: number; + linkCount: number; + mode: number; +} + +export interface RenameIdentityResult { + source: NamespaceEntryIdentity; + replaced?: NamespaceEntryIdentity; +} + interface DirIndexEntry { ino: number; abs: number; @@ -156,6 +184,7 @@ interface DirIndexEntry { interface DirIndex { generation: number; + mutationSequence: number; size: number; entries: Map; free: Array<{ abs: number; recLen: number }>; @@ -165,11 +194,13 @@ const ERROR_MESSAGES: Record = { [ENOENT]: "No such file or directory", [EIO]: "I/O error", [EBADF]: "Bad file descriptor", + [EBUSY]: "Device or resource busy", [EEXIST]: "File exists", [ENOTDIR]: "Not a directory", [EISDIR]: "Is a directory", [EINVAL]: "Invalid argument", [EMFILE]: "Too many open files", + [EFBIG]: "File too large", [ENOSPC]: "No space left on device", [ENAMETOOLONG]: "File name too long", [ENOTEMPTY]: "Directory not empty", @@ -192,6 +223,10 @@ const encoder = new TextEncoder(); const decoder = new TextDecoder(); const DOTDOT_BYTES = encoder.encode(".."); +function isReservedDirectoryName(name: string): boolean { + return name === "." || name === ".."; +} + /** * Safely decode a Uint8Array that may be backed by SharedArrayBuffer. * Browsers reject SAB-backed views in TextDecoder.decode(). @@ -216,13 +251,14 @@ export class SharedFS { private dirIndexes = new Map(); private blockAllocHint = 0; private inodeAllocHint = 2; + private atomicsWaitAllowed: boolean | undefined; /** * Directory operations are stored in ext2-style variable-length entries. - * Linear scans are fine for normal directories, but workloads such as - * PHP's bug36365 test create tens of thousands of files in one directory. - * Build an in-memory name index once a directory reaches this size so - * creates/stat lookups stay near O(1) instead of O(n²). + * Workloads such as PHP's bug36365 test create tens of thousands of files in + * one directory. Once a directory reaches this threshold, retain validated + * entry locations so each repeated exact-name lookup does not rescan every + * preceding variable-length record. */ private static readonly DIR_INDEX_MIN_SIZE = 64 * 1024; @@ -246,14 +282,11 @@ export class SharedFS { // Size inodes for max capacity so we don't run out after growth let totalInodes = Math.floor(maxBlocks / 4); if (totalInodes < 32) totalInodes = 32; - totalInodes = - Math.ceil(totalInodes / INODES_PER_BLOCK) * INODES_PER_BLOCK; + totalInodes = Math.ceil(totalInodes / INODES_PER_BLOCK) * INODES_PER_BLOCK; const inodeBitmapBlocks = Math.ceil(totalInodes / (BLOCK_SIZE * 8)); const blockBitmapBlocks = Math.ceil(maxBlocks / (BLOCK_SIZE * 8)); - const inodeTableBlocks = Math.ceil( - (totalInodes * INODE_SIZE) / BLOCK_SIZE, - ); + const inodeTableBlocks = Math.ceil((totalInodes * INODE_SIZE) / BLOCK_SIZE); const inodeBitmapStart = 1; const blockBitmapStart = inodeBitmapStart + inodeBitmapBlocks; @@ -353,17 +386,165 @@ export class SharedFS { return fs; } - static mount(buffer: SharedArrayBuffer): SharedFS { + static mount( + buffer: SharedArrayBuffer, + options?: { restoreImage?: boolean }, + ): SharedFS { const fs = new SharedFS(buffer); if (fs.r32(SB_MAGIC) !== MAGIC) throw new SFSError(EINVAL, "Bad magic"); if (fs.r32(SB_VERSION) !== VERSION) throw new SFSError(EINVAL, "Bad version"); if (fs.r32(SB_BLOCK_SIZE) !== BLOCK_SIZE) throw new SFSError(EINVAL, "Bad block size"); + if (options?.restoreImage) fs.resetRestoredRuntimeState(); fs.resetAllocationHints(); return fs; } + /** + * Return a portable, quiescent copy of the filesystem bytes. + * + * File descriptors and inode locks are process-runtime state, not VFS image + * state. Refuse to snapshot while descriptors are live, then clear all lock + * words in the copy so a restored image cannot inherit a dead worker's lock. + */ + snapshotBytes(): Uint8Array { + return this.withNamespaceLock(() => this.snapshotBytesUnlocked()); + } + + snapshotState(): { + bytes: Uint8Array; + identities: Map; + } { + return this.withNamespaceLock(() => { + // Validate quiescence and copy first. With the namespace lock held, no + // new descriptor can appear while the matching path identities are + // collected for lazy metadata. + const bytes = this.snapshotBytesUnlocked(); + return { bytes, identities: this.collectIdentityStateUnlocked() }; + }); + } + + identityState(): Map { + return this.withNamespaceLock(() => this.collectIdentityStateUnlocked()); + } + + private snapshotBytesUnlocked(): Uint8Array { + for (let fd = 0; fd < MAX_FDS; fd++) { + const base = FD_TABLE_OFFSET + fd * FD_ENTRY_SIZE; + if (Atomics.load(this.i32, base >> 2) !== 0) { + throw new SFSError( + EBUSY, + "Cannot save a VFS image with open descriptors", + ); + } + } + + const totalInodes = this.r32(SB_TOTAL_INODES); + for (let ino = 0; ino < totalInodes; ino++) { + const off = this.inodeOffset(ino); + if (this.r32(off + INO_OPEN_COUNT) !== 0) { + throw new SFSError( + EBUSY, + "Cannot save a VFS image with open inode references", + ); + } + } + + const copy = new Uint8Array(this.buffer.byteLength); + copy.set(this.u8); + const view = new DataView(copy.buffer); + view.setUint32(SB_GLOBAL_LOCK, 0, true); + view.setUint32(SB_NAMESPACE_LOCK, 0, true); + copy.fill(0, FD_TABLE_OFFSET, BLOCK_SIZE); + + for (let ino = 0; ino < totalInodes; ino++) { + const off = this.inodeOffset(ino); + view.setUint32(off + INO_LOCK_STATE, 0, true); + view.setUint32(off + INO_OPEN_COUNT, 0, true); + } + return copy; + } + + private collectIdentityStateUnlocked(): Map { + const identities = new Map(); + const directories: Array<{ ino: number; path: string }> = [ + { ino: ROOT_INO, path: "/" }, + ]; + const visitedDirectories = new Set(); + + while (directories.length > 0) { + const directory = directories.pop()!; + if (visitedDirectories.has(directory.ino)) throw new SFSError(EIO); + visitedDirectories.add(directory.ino); + + const inoOff = this.inodeOffset(directory.ino); + if ((this.r32(inoOff + INO_MODE) & S_IFMT) !== S_IFDIR) { + throw new SFSError(EIO); + } + const dirSize = this.r64(inoOff + INO_SIZE); + let pos = 0; + while (pos < dirSize) { + const fileBlock = Math.floor(pos / BLOCK_SIZE); + const blockOff = pos % BLOCK_SIZE; + const phys = this.inodeBlockMap(directory.ino, fileBlock, false); + if (phys <= 0) throw new SFSError(EIO); + const blockBase = phys * BLOCK_SIZE; + const remain = Math.min(dirSize - pos, BLOCK_SIZE - blockOff); + + let off = blockOff; + while (off < blockOff + remain) { + const abs = blockBase + off; + const entIno = this.r32(abs); + const recLen = this.view.getUint16(abs + 4, true); + const nameLen = this.view.getUint16(abs + 6, true); + if (!this.isValidDirEntry(off, blockOff + remain, recLen, nameLen)) { + throw new SFSError(EIO); + } + if (entIno !== 0) { + if (!this.inodeIsAllocated(entIno)) throw new SFSError(EIO); + const name = safeDecode( + this.u8.subarray( + abs + DIRENT_HEADER_SIZE, + abs + DIRENT_HEADER_SIZE + nameLen, + ), + ); + if (name !== "." && name !== "..") { + const childPath = + directory.path === "/" + ? `/${name}` + : `${directory.path}/${name}`; + const childOff = this.inodeOffset(entIno); + const generation = this.r64(childOff + INO_GENERATION); + const key = `${entIno}:${generation}`; + let identity = identities.get(key); + if (!identity) { + identity = { + ino: entIno, + generation, + dataSequence: + Atomics.load( + this.i32, + (childOff + INO_DATA_SEQUENCE) >> 2, + ) >>> 0, + paths: [], + }; + identities.set(key, identity); + } + identity.paths.push(childPath); + if ((this.r32(childOff + INO_MODE) & S_IFMT) === S_IFDIR) { + directories.push({ ino: entIno, path: childPath }); + } + } + } + off += recLen; + } + pos += remain; + } + } + return identities; + } + statfs(): SharedFsStats { const blockSize = this.r32(SB_BLOCK_SIZE); const currentBlocks = this.r32(SB_TOTAL_BLOCKS); @@ -405,6 +586,32 @@ export class SharedFS { this.view.setBigUint64(off, BigInt(v), true); } + /** + * Wait for a shared lock word to change. + * + * Browser main threads forbid Atomics.wait(). The legacy shared-filesystem + * BrowserKernel path still exposes synchronous MemoryFileSystem methods on + * that thread, so fall back to atomic polling while the independently + * scheduled kernel worker finishes its short critical section. Preferred + * kernel-owned browser boots never exercise this fallback. + */ + private waitForAtomicChange(index: number, expected: number): void { + if (this.atomicsWaitAllowed !== false) { + try { + Atomics.wait(this.i32, index, expected); + this.atomicsWaitAllowed = true; + return; + } catch (error) { + if (!(error instanceof TypeError)) throw error; + this.atomicsWaitAllowed = false; + } + } + while (Atomics.load(this.i32, index) === expected) { + // Synchronous SharedFS cannot yield a Promise here. The lock owner is a + // different worker, so atomic polling does not prevent its progress. + } + } + private resetAllocationHints(): void { this.blockAllocHint = this.findNextFreeBlockHint(); this.inodeAllocHint = this.findNextFreeInodeHint(); @@ -440,7 +647,7 @@ export class SharedFS { for (;;) { const old = Atomics.compareExchange(this.i32, idx, 0, 1); if (old === 0) return; - Atomics.wait(this.i32, idx, 1); + this.waitForAtomicChange(idx, 1); } } @@ -450,6 +657,67 @@ export class SharedFS { Atomics.notify(this.i32, idx, Infinity); } + // ── Namespace lock (path resolution and mutation) ─────────────── + + private namespaceLock(): void { + const idx = SB_NAMESPACE_LOCK >> 2; + for (;;) { + const old = Atomics.compareExchange(this.i32, idx, 0, 1); + if (old === 0) return; + this.waitForAtomicChange(idx, 1); + } + } + + private namespaceUnlock(): void { + const idx = SB_NAMESPACE_LOCK >> 2; + Atomics.store(this.i32, idx, 0); + Atomics.notify(this.i32, idx, Infinity); + } + + private withNamespaceLock(operation: () => T): T { + this.namespaceLock(); + try { + return operation(); + } finally { + this.namespaceUnlock(); + } + } + + /** Reset process-local runtime state after copying a portable image. */ + private resetRestoredRuntimeState(): void { + // The buffer is private to fromImage(), so stale locks can be cleared + // directly before any lock-taking operation is attempted. + Atomics.store(this.i32, SB_GLOBAL_LOCK >> 2, 0); + Atomics.store(this.i32, SB_NAMESPACE_LOCK >> 2, 0); + this.u8.fill(0, FD_TABLE_OFFSET, BLOCK_SIZE); + + const totalInodes = this.r32(SB_TOTAL_INODES); + const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; + for (let ino = 0; ino < totalInodes; ino++) { + const off = this.inodeOffset(ino); + this.w32(off + INO_LOCK_STATE, 0); + this.w32(off + INO_OPEN_COUNT, 0); + if (ino < 2) continue; + const word = this.r32(ibStart + (ino >> 5) * 4); + if ((word & (1 << (ino & 31))) === 0) continue; + if (this.r32(off + INO_LINK_COUNT) !== 0) continue; + + const mode = this.r32(off + INO_MODE); + const size = this.r64(off + INO_SIZE); + if ((mode & S_IFMT) === S_IFLNK && size <= INLINE_SYMLINK_SIZE) { + this.u8.fill( + 0, + off + INO_DIRECT, + off + INO_DIRECT + INLINE_SYMLINK_SIZE, + ); + this.w64(off + INO_SIZE, 0); + } else { + this.inodeTruncate(ino, 0); + } + this.inodeFree(ino); + } + } + // ── Block allocator ────────────────────────────────────────────── private blockAlloc(): number { @@ -607,6 +875,7 @@ export class SharedFS { for (;;) { const word = Atomics.load(this.i32, idx); + if ((word & (1 << bit)) === 0) throw new SFSError(EIO); const desired = word & ~(1 << bit); const old = Atomics.compareExchange(this.i32, idx, word, desired); if (old === word) break; @@ -615,9 +884,16 @@ export class SharedFS { if (ino >= 2 && ino < this.inodeAllocHint) this.inodeAllocHint = ino; } - private inodeAddOpenRef(ino: number): void { - const idx = (this.inodeOffset(ino) + INO_OPEN_COUNT) >> 2; - Atomics.add(this.i32, idx, 1); + private inodeAddOpenRef(ino: number): boolean { + this.inodeWriteLock(ino); + try { + const off = this.inodeOffset(ino); + if (this.r32(off + INO_LINK_COUNT) === 0) return false; + this.w32(off + INO_OPEN_COUNT, this.r32(off + INO_OPEN_COUNT) + 1); + return true; + } finally { + this.inodeWriteUnlock(ino); + } } private inodeDropOpenRef(ino: number): void { @@ -676,15 +952,10 @@ export class SharedFS { for (;;) { const cur = Atomics.load(this.i32, lockIdx); if (cur & WRITER_BIT) { - Atomics.wait(this.i32, lockIdx, cur); + this.waitForAtomicChange(lockIdx, cur); continue; } - const old = Atomics.compareExchange( - this.i32, - lockIdx, - cur, - cur + 1, - ); + const old = Atomics.compareExchange(this.i32, lockIdx, cur, cur + 1); if (old === cur) return; } } @@ -702,15 +973,10 @@ export class SharedFS { for (;;) { const cur = Atomics.load(this.i32, lockIdx); if (cur !== 0) { - Atomics.wait(this.i32, lockIdx, cur); + this.waitForAtomicChange(lockIdx, cur); continue; } - const old = Atomics.compareExchange( - this.i32, - lockIdx, - 0, - WRITER_BIT, - ); + const old = Atomics.compareExchange(this.i32, lockIdx, 0, WRITER_BIT); if (old === 0) return; } } @@ -745,18 +1011,26 @@ export class SharedFS { fileBlock -= DIRECT_BLOCKS; if (fileBlock < PTRS_PER_BLOCK) { let ind = this.r32(inoOff + INO_INDIRECT); + let allocatedIndirect = false; if (ind === 0) { if (!allocate) return 0; ind = this.blockAllocWithGrow(); if (ind < 0) return ind; this.w32(inoOff + INO_INDIRECT, ind); + allocatedIndirect = true; } const ptrOff = ind * BLOCK_SIZE + fileBlock * 4; const ptr = this.r32(ptrOff); if (ptr !== 0) return ptr; if (!allocate) return 0; const blk = this.blockAllocWithGrow(); - if (blk < 0) return blk; + if (blk < 0) { + if (allocatedIndirect) { + this.w32(inoOff + INO_INDIRECT, 0); + this.blockFree(ind); + } + return blk; + } this.w32(ptrOff, blk); return blk; } @@ -768,20 +1042,30 @@ export class SharedFS { const idx2 = fileBlock % PTRS_PER_BLOCK; let dind = this.r32(inoOff + INO_DOUBLE_INDIRECT); + let allocatedDoubleIndirect = false; if (dind === 0) { if (!allocate) return 0; dind = this.blockAllocWithGrow(); if (dind < 0) return dind; this.w32(inoOff + INO_DOUBLE_INDIRECT, dind); + allocatedDoubleIndirect = true; } const l1Off = dind * BLOCK_SIZE + idx1 * 4; let l1 = this.r32(l1Off); + let allocatedFirstLevel = false; if (l1 === 0) { if (!allocate) return 0; l1 = this.blockAllocWithGrow(); - if (l1 < 0) return l1; + if (l1 < 0) { + if (allocatedDoubleIndirect) { + this.w32(inoOff + INO_DOUBLE_INDIRECT, 0); + this.blockFree(dind); + } + return l1; + } this.w32(l1Off, l1); + allocatedFirstLevel = true; } const l2Off = l1 * BLOCK_SIZE + idx2 * 4; @@ -789,7 +1073,17 @@ export class SharedFS { if (ptr !== 0) return ptr; if (!allocate) return 0; const blk = this.blockAllocWithGrow(); - if (blk < 0) return blk; + if (blk < 0) { + if (allocatedFirstLevel) { + this.w32(l1Off, 0); + this.blockFree(l1); + } + if (allocatedDoubleIndirect) { + this.w32(inoOff + INO_DOUBLE_INDIRECT, 0); + this.blockFree(dind); + } + return blk; + } this.w32(l2Off, blk); return blk; } @@ -844,7 +1138,7 @@ export class SharedFS { const inoOff = this.inodeOffset(ino); const size = this.r64(inoOff + INO_SIZE); if (offset > size) { - this.zeroInodeRange(ino, size, offset); + this.zeroOldEofTail(ino, size); } let totalWritten = 0; @@ -857,7 +1151,10 @@ export class SharedFS { if (chunk > count) chunk = count; const phys = this.inodeBlockMap(ino, fileBlock, true); - if (phys < 0) return totalWritten > 0 ? totalWritten : phys; + if (phys < 0) { + if (totalWritten === 0) return phys; + break; + } const dstOff = phys * BLOCK_SIZE + blockOff; this.u8.set(src.subarray(srcPos, srcPos + chunk), dstOff); @@ -868,13 +1165,14 @@ export class SharedFS { totalWritten += chunk; } - if (offset > this.r64(inoOff + INO_SIZE)) { + if (totalWritten > 0 && offset > this.r64(inoOff + INO_SIZE)) { this.w64(inoOff + INO_SIZE, offset); } if (totalWritten > 0) { const now = Date.now(); this.w64(inoOff + INO_MTIME, now); this.w64(inoOff + INO_CTIME, now); + Atomics.add(this.i32, (inoOff + INO_DATA_SEQUENCE) >> 2, 1); } return totalWritten; } @@ -893,6 +1191,21 @@ export class SharedFS { } } + /** + * Zero only the allocated tail of the old EOF block. Sparse extension does + * not need to walk logical holes: absent blocks already read as zero and a + * newly allocated block is cleared by blockAlloc(). + */ + private zeroOldEofTail(ino: number, oldSize: number): void { + const blockOff = oldSize % BLOCK_SIZE; + if (blockOff === 0) return; + const fileBlock = Math.floor(oldSize / BLOCK_SIZE); + const phys = this.inodeBlockMap(ino, fileBlock, false); + if (phys <= 0) return; + const start = phys * BLOCK_SIZE + blockOff; + this.u8.fill(0, start, phys * BLOCK_SIZE + BLOCK_SIZE); + } + private freeBlocksFrom(ino: number, fromBlock: number): void { const inoOff = this.inodeOffset(ino); @@ -908,8 +1221,7 @@ export class SharedFS { // Single indirect const ind = this.r32(inoOff + INO_INDIRECT); if (ind) { - const start = - fromBlock > DIRECT_BLOCKS ? fromBlock - DIRECT_BLOCKS : 0; + const start = fromBlock > DIRECT_BLOCKS ? fromBlock - DIRECT_BLOCKS : 0; for (let i = start; i < PTRS_PER_BLOCK; i++) { const ptrOff = ind * BLOCK_SIZE + i * 4; const ptr = this.r32(ptrOff); @@ -959,19 +1271,24 @@ export class SharedFS { } } - private inodeTruncate(ino: number, newSize: number): void { + private inodeTruncate( + ino: number, + newSize: number, + forceDataMutation = false, + ): void { const inoOff = this.inodeOffset(ino); const curSize = this.r64(inoOff + INO_SIZE); const sizeChanged = newSize !== curSize; if (newSize >= curSize) { if (newSize > curSize) { - this.zeroInodeRange(ino, curSize, newSize); + this.zeroOldEofTail(ino, curSize); } this.w64(inoOff + INO_SIZE, newSize); - if (sizeChanged) { + if (sizeChanged || forceDataMutation) { const now = Date.now(); this.w64(inoOff + INO_MTIME, now); this.w64(inoOff + INO_CTIME, now); + Atomics.add(this.i32, (inoOff + INO_DATA_SEQUENCE) >> 2, 1); } return; } @@ -985,13 +1302,19 @@ export class SharedFS { const keepBlocks = Math.ceil(newSize / BLOCK_SIZE); this.freeBlocksFrom(ino, keepBlocks); this.w64(inoOff + INO_SIZE, newSize); - if (sizeChanged) { + if (sizeChanged || forceDataMutation) { const now = Date.now(); this.w64(inoOff + INO_MTIME, now); this.w64(inoOff + INO_CTIME, now); + Atomics.add(this.i32, (inoOff + INO_DATA_SEQUENCE) >> 2, 1); } } + private validateFileSize(size: number): void { + if (!Number.isSafeInteger(size) || size < 0) throw new SFSError(EINVAL); + if (size > MAX_FILE_SIZE) throw new SFSError(EFBIG); + } + // ── Directory operations ───────────────────────────────────────── private touchDirectoryMutation(dirIno: number): void { @@ -999,6 +1322,13 @@ export class SharedFS { const now = Date.now(); this.w64(inoOff + INO_MTIME, now); this.w64(inoOff + INO_CTIME, now); + const mutationSequence = + (Atomics.add(this.i32, (inoOff + INO_DIR_SEQUENCE) >> 2, 1) + 1) >>> 0; + const index = this.dirIndexes.get(dirIno); + if (index) { + index.mutationSequence = mutationSequence; + index.size = this.r64(inoOff + INO_SIZE); + } } private dirNameKey(name: Uint8Array): string { @@ -1014,9 +1344,32 @@ export class SharedFS { return true; } + private isValidDirEntry( + off: number, + endOff: number, + recLen: number, + nameLen: number, + ): boolean { + return ( + recLen >= DIRENT_HEADER_SIZE && + recLen % 4 === 0 && + off + recLen <= endOff && + nameLen <= recLen - DIRENT_HEADER_SIZE + ); + } + + private inodeIsAllocated(ino: number): boolean { + const totalInodes = this.r32(SB_TOTAL_INODES); + if (ino <= 0 || ino >= totalInodes) return false; + const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; + const word = Atomics.load(this.i32, (ibStart >> 2) + (ino >> 5)); + return (word & (1 << (ino & 31))) !== 0; + } + private rebuildDirIndex( dirIno: number, generation: number, + mutationSequence: number, dirSize: number, ): DirIndex | number { const entries = new Map(); @@ -1040,9 +1393,11 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return EIO; + if (!this.isValidDirEntry(off, blockOff + remain, recLen, entNameLen)) + return EIO; if (entIno !== 0) { + if (!this.inodeIsAllocated(entIno)) return EIO; const name = safeDecode( this.u8.subarray( abs + DIRENT_HEADER_SIZE, @@ -1064,7 +1419,13 @@ export class SharedFS { pos += remain; } - const index = { generation, size: dirSize, entries, free }; + const index = { + generation, + mutationSequence, + size: dirSize, + entries, + free, + }; this.dirIndexes.set(dirIno, index); return index; } @@ -1073,10 +1434,13 @@ export class SharedFS { const inoOff = this.inodeOffset(dirIno); const dirSize = this.r64(inoOff + INO_SIZE); const generation = this.r64(inoOff + INO_GENERATION); + const mutationSequence = + Atomics.load(this.i32, (inoOff + INO_DIR_SEQUENCE) >> 2) >>> 0; const cached = this.dirIndexes.get(dirIno); if ( cached && cached.generation === generation && + cached.mutationSequence === mutationSequence && cached.size === dirSize ) { return cached; @@ -1084,7 +1448,7 @@ export class SharedFS { if (cached) this.dirIndexes.delete(dirIno); if (dirSize < SharedFS.DIR_INDEX_MIN_SIZE) return null; - return this.rebuildDirIndex(dirIno, generation, dirSize); + return this.rebuildDirIndex(dirIno, generation, mutationSequence, dirSize); } private updateDirIndexAdd( @@ -1181,6 +1545,7 @@ export class SharedFS { // in-process indexes cannot resurrect an externally removed name. if ( this.r32(entry.abs) === entry.ino && + this.inodeIsAllocated(entry.ino) && this.view.getUint16(entry.abs + 4, true) === entry.recLen && this.view.getUint16(entry.abs + 6, true) === entry.nameLen && this.dirEntryNameMatches(entry.abs, name) @@ -1213,7 +1578,8 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return EIO; + if (!this.isValidDirEntry(off, blockOff + remain, recLen, entNameLen)) + return EIO; if (entIno !== 0 && entNameLen === name.length) { let match = true; @@ -1223,7 +1589,10 @@ export class SharedFS { break; } } - if (match) return entIno; + if (match) { + if (!this.inodeIsAllocated(entIno)) return EIO; + return entIno; + } } off += recLen; } @@ -1245,7 +1614,13 @@ export class SharedFS { while (off < endOff) { const abs = blockBase + off; const recLen = this.view.getUint16(abs + 4, true); - if (recLen === 0 || off + recLen > endOff) return -1; + if ( + recLen < DIRENT_HEADER_SIZE || + recLen % 4 !== 0 || + off + recLen > endOff + ) { + return -1; + } lastAbs = abs; off += recLen; } @@ -1267,19 +1642,15 @@ export class SharedFS { let appendPos = dirSize; let fileBlock = Math.floor(appendPos / BLOCK_SIZE); let blockOff = appendPos % BLOCK_SIZE; + let reservedPhys = 0; if (blockOff !== 0 && blockOff + needed > BLOCK_SIZE) { // Entry doesn't fit in remaining space — skip to next block. const gap = BLOCK_SIZE - blockOff; + let padPhys = 0; if (gap >= DIRENT_HEADER_SIZE) { - // Write a padding entry (ino=0) to fill the gap - const padPhys = this.inodeBlockMap(dirIno, fileBlock, false); - if (padPhys > 0) { - const padAbs = padPhys * BLOCK_SIZE + blockOff; - this.w32(padAbs, 0); - this.view.setUint16(padAbs + 4, gap, true); - this.view.setUint16(padAbs + 6, 0, true); - } + padPhys = this.inodeBlockMap(dirIno, fileBlock, false); + if (padPhys <= 0) return EIO; } else { if (lastEntAbs < 0) { lastEntAbs = this.findLastDirEntryInBlock( @@ -1288,13 +1659,25 @@ export class SharedFS { blockOff, ); } - if (lastEntAbs >= 0) { - // Gap too small for a padding entry — extend last entry's recLen - const oldRecLen = this.view.getUint16(lastEntAbs + 4, true); - const newRecLen = oldRecLen + gap; - this.view.setUint16(lastEntAbs + 4, newRecLen, true); - this.updateDirIndexRecLen(dirIno, lastEntAbs, newRecLen); - } + if (lastEntAbs < 0) return EIO; + } + + // Reserve the destination block before changing the old tail. If the + // allocation fails, the directory remains byte-for-byte unchanged. + reservedPhys = this.inodeBlockMap(dirIno, fileBlock + 1, true); + if (reservedPhys < 0) return reservedPhys; + if (gap >= DIRENT_HEADER_SIZE) { + // Write a padding entry (ino=0) to fill the gap + const padAbs = padPhys * BLOCK_SIZE + blockOff; + this.w32(padAbs, 0); + this.view.setUint16(padAbs + 4, gap, true); + this.view.setUint16(padAbs + 6, 0, true); + } else { + // Gap too small for a padding entry — extend last entry's recLen + const oldRecLen = this.view.getUint16(lastEntAbs + 4, true); + const newRecLen = oldRecLen + gap; + this.view.setUint16(lastEntAbs + 4, newRecLen, true); + this.updateDirIndexRecLen(dirIno, lastEntAbs, newRecLen); } appendPos = (fileBlock + 1) * BLOCK_SIZE; fileBlock++; @@ -1304,7 +1687,7 @@ export class SharedFS { // Need a new block? let phys: number; if (blockOff === 0) { - phys = this.inodeBlockMap(dirIno, fileBlock, true); + phys = reservedPhys || this.inodeBlockMap(dirIno, fileBlock, true); if (phys < 0) return phys; } else { phys = this.inodeBlockMap(dirIno, fileBlock, false); @@ -1333,9 +1716,8 @@ export class SharedFS { if (index) { if (this.useDirIndexFreeSlot(index, dirIno, name, childIno)) return 0; - // Large indexed directories favor append-only growth when no indexed - // deleted slot is available. A full scan to discover slack on every - // create would reintroduce the O(n²) behavior the index avoids. + // When no indexed deleted slot is available, append instead of scanning + // every existing record again solely to discover internal slack. return this.dirAppendEntry(dirIno, name, childIno); } @@ -1365,7 +1747,13 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return EIO; + if ( + recLen < DIRENT_HEADER_SIZE || + recLen % 4 !== 0 || + off + recLen > blockOff + remain || + entNameLen > recLen - DIRENT_HEADER_SIZE + ) + return EIO; if (entIno === 0 && recLen >= needed) { // Reuse deleted entry @@ -1448,7 +1836,8 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return EIO; + if (!this.isValidDirEntry(off, blockOff + remain, recLen, entNameLen)) + return EIO; if (entIno !== 0 && entNameLen === name.length) { let match = true; @@ -1492,6 +1881,7 @@ export class SharedFS { ) { this.w32(entry.abs, childIno); entry.ino = childIno; + this.touchDirectoryMutation(dirIno); return 0; } @@ -1520,7 +1910,8 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return EIO; + if (!this.isValidDirEntry(off, blockOff + remain, recLen, entNameLen)) + return EIO; if (entIno !== 0 && entNameLen === name.length) { let match = true; @@ -1532,6 +1923,7 @@ export class SharedFS { } if (match) { this.w32(abs, childIno); + this.touchDirectoryMutation(dirIno); this.updateDirIndexAdd(dirIno, name, childIno, abs, recLen); return 0; } @@ -1552,7 +1944,7 @@ export class SharedFS { const fileBlock = Math.floor(pos / BLOCK_SIZE); const blockOff = pos % BLOCK_SIZE; const phys = this.inodeBlockMap(dirIno, fileBlock, false); - if (phys <= 0) return true; + if (phys <= 0) throw new SFSError(EIO); const blockBase = phys * BLOCK_SIZE; let remain = dirSize - pos; @@ -1565,12 +1957,17 @@ export class SharedFS { const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) break; + if ( + recLen < DIRENT_HEADER_SIZE || + recLen % 4 !== 0 || + off + recLen > blockOff + remain || + entNameLen > recLen - DIRENT_HEADER_SIZE + ) + throw new SFSError(EIO); if (entIno !== 0) { // Skip "." and ".." - if (entNameLen === 1 && this.u8[abs + DIRENT_HEADER_SIZE] === 0x2e) - { + if (entNameLen === 1 && this.u8[abs + DIRENT_HEADER_SIZE] === 0x2e) { off += recLen; continue; } @@ -1599,11 +1996,11 @@ export class SharedFS { if (cur === ROOT_INO) return false; const parent = this.dirLookup(cur, DOTDOT_BYTES); - if (parent < 0 || parent === cur) return false; + if (parent < 0 || parent === cur) throw new SFSError(EIO); cur = parent; } - return true; + throw new SFSError(EIO); } // ── Path resolution ────────────────────────────────────────────── @@ -1620,12 +2017,17 @@ export class SharedFS { const part = parts[pi]; if (part.length > MAX_NAME) return ENAMETOOLONG; - const inoOff = this.inodeOffset(ino); - const mode = this.r32(inoOff + INO_MODE); - if ((mode & S_IFMT) !== S_IFDIR) return ENOTDIR; - const nameBytes = encoder.encode(part); - const childIno = this.dirLookup(ino, nameBytes); + let childIno: number; + this.inodeReadLock(ino); + try { + const inoOff = this.inodeOffset(ino); + const mode = this.r32(inoOff + INO_MODE); + if ((mode & S_IFMT) !== S_IFDIR) return ENOTDIR; + childIno = this.dirLookup(ino, nameBytes); + } finally { + this.inodeReadUnlock(ino); + } if (childIno < 0) return childIno; // Check if child is symlink @@ -1657,18 +2059,14 @@ export class SharedFS { if (target.startsWith("/")) { // Absolute symlink — restart from root ino = ROOT_INO; - const targetParts = target - .split("/") - .filter((p) => p.length > 0); + const targetParts = target.split("/").filter((p) => p.length > 0); const remaining = parts.slice(pi + 1); parts.length = 0; parts.push(...targetParts, ...remaining); pi = -1; // will be incremented to 0 } else { // Relative symlink — splice into remaining path - const targetParts = target - .split("/") - .filter((p) => p.length > 0); + const targetParts = target.split("/").filter((p) => p.length > 0); const remaining = parts.slice(pi + 1); parts.length = pi; parts.push(...targetParts, ...remaining); @@ -1720,16 +2118,17 @@ export class SharedFS { this.w64(base + FD_OFFSET, 0); this.w32(base + FD_FLAGS, flags); this.w32(base + FD_IS_DIR, isDir ? 1 : 0); - this.inodeAddOpenRef(ino); + if (!this.inodeAddOpenRef(ino)) { + Atomics.store(this.i32, idx, 0); + return ENOENT; + } return i; } } return EMFILE; } - private fdGet( - fd: number, - ): { + private fdGet(fd: number): { base: number; ino: number; offset: number; @@ -1763,6 +2162,7 @@ export class SharedFS { return { ino, generation: this.r64(off + INO_GENERATION), + dataSequence: this.r32(off + INO_DATA_SEQUENCE), mode: this.r32(off + INO_MODE), linkCount: this.r32(off + INO_LINK_COUNT), size: this.r64(off + INO_SIZE), @@ -1774,9 +2174,121 @@ export class SharedFS { }; } + private namespaceEntryIdentity(ino: number): NamespaceEntryIdentity { + const off = this.inodeOffset(ino); + return { + ino, + generation: this.r64(off + INO_GENERATION), + linkCount: this.r32(off + INO_LINK_COUNT), + mode: this.r32(off + INO_MODE), + }; + } + // ── Public API: File operations ────────────────────────────────── open(path: string, flags: number, createMode: number = 0o644): number { + return this.withNamespaceLock(() => + this.openUnlocked(path, flags, createMode), + ); + } + + /** + * Atomically create or truncate an empty lazy-file stub and capture the + * exact data identity produced by that truncation. Existing callers rely on + * lazy registration replacing a path, while the inode lock ensures a peer + * write either precedes the truncation or advances the captured sequence + * after it. + */ + createLazyStub(path: string, mode: number): StatResult { + return this.withNamespaceLock(() => { + const fd = this.openUnlocked(path, O_WRONLY | O_CREAT, mode); + try { + const entry = this.fdGet(fd); + if (!entry) throw new SFSError(EBADF); + this.inodeWriteLock(entry.ino); + try { + this.inodeTruncate(entry.ino, 0, true); + return this.buildStat(entry.ino); + } finally { + this.inodeWriteUnlock(entry.ino); + } + } finally { + this.closeUnlocked(fd); + } + }); + } + + /** + * Atomically replace a lazy stub only if the path still names the exact + * inode content generation observed before an asynchronous fetch. + */ + replaceIfIdentity( + path: string, + expectedIno: number, + expectedGeneration: number, + expectedDataSequence: number, + data: Uint8Array, + ): boolean { + return this.withNamespaceLock(() => { + // Lazy access follows symlinks, so conditionally replace the resolved + // regular target while still validating its exact inode/data identity. + const ino = this.pathResolve(path, true); + if (ino < 0 || ino !== expectedIno) return false; + const off = this.inodeOffset(ino); + if ( + this.r64(off + INO_GENERATION) !== expectedGeneration || + this.r32(off + INO_DATA_SEQUENCE) !== expectedDataSequence + ) + return false; + if ((this.r32(off + INO_MODE) & S_IFMT) !== S_IFREG) return false; + this.validateFileSize(data.byteLength); + + this.inodeWriteLock(ino); + try { + // Descriptor-based writes do not take the namespace lock. Revalidate + // after acquiring the inode lock so a concurrent guest mutation that + // won the race cannot be overwritten by stale fetched bytes. + if ( + this.r64(off + INO_GENERATION) !== expectedGeneration || + this.r32(off + INO_DATA_SEQUENCE) !== expectedDataSequence + ) + return false; + // Lazy backing is attached only to an untouched empty stub. Refuse to + // replace any concrete content even if malformed metadata happens to + // carry its current sequence. + if (this.r64(off + INO_SIZE) !== 0) return false; + + const originalMtime = this.r64(off + INO_MTIME); + const originalCtime = this.r64(off + INO_CTIME); + + this.inodeTruncate(ino, 0, true); + const written = + data.byteLength > 0 + ? this.inodeWriteData(ino, 0, data, data.byteLength) + : 0; + if (written !== data.byteLength) { + this.inodeTruncate(ino, 0, true); + Atomics.store( + this.i32, + (off + INO_DATA_SEQUENCE) >> 2, + expectedDataSequence, + ); + this.w64(off + INO_MTIME, originalMtime); + this.w64(off + INO_CTIME, originalCtime); + throw new SFSError(written < 0 ? written : ENOSPC); + } + return true; + } finally { + this.inodeWriteUnlock(ino); + } + }); + } + + private openUnlocked( + path: string, + flags: number, + createMode: number = 0o644, + ): number { const accMode = flags & O_ACCMODE; const creating = (flags & O_CREAT) !== 0; const exclusive = (flags & O_EXCL) !== 0; @@ -1835,7 +2347,7 @@ export class SharedFS { } // O_DIRECTORY: reject non-directories - if ((flags & O_DIRECTORY) && (mode & S_IFMT) !== S_IFDIR) { + if (flags & O_DIRECTORY && (mode & S_IFMT) !== S_IFDIR) { throw new SFSError(ENOTDIR); } @@ -1843,7 +2355,7 @@ export class SharedFS { if (flags & O_TRUNC) { if ((mode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); this.inodeWriteLock(ino); - this.inodeTruncate(ino, 0); + this.inodeTruncate(ino, 0, true); this.inodeWriteUnlock(ino); } @@ -1854,6 +2366,10 @@ export class SharedFS { } close(fd: number): void { + this.withNamespaceLock(() => this.closeUnlocked(fd)); + } + + private closeUnlocked(fd: number): void { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); this.fdFree(fd); @@ -1898,6 +2414,12 @@ export class SharedFS { const inoOff = this.inodeOffset(entry.ino); offset = this.r64(inoOff + INO_SIZE); } + if (!Number.isSafeInteger(offset) || offset < 0) { + throw new SFSError(EINVAL); + } + if (offset > MAX_FILE_SIZE || data.length > MAX_FILE_SIZE - offset) { + throw new SFSError(EFBIG); + } const nwritten = this.inodeWriteData( entry.ino, @@ -1905,6 +2427,7 @@ export class SharedFS { data, data.length, ); + if (nwritten < 0) return nwritten; // Update offset const base = FD_TABLE_OFFSET + fd * FD_ENTRY_SIZE; this.w64(base + FD_OFFSET, offset + nwritten); @@ -1931,7 +2454,7 @@ export class SharedFS { throw new SFSError(EINVAL); } - if (newOffset < 0) throw new SFSError(EINVAL); + this.validateFileSize(newOffset); const base = FD_TABLE_OFFSET + fd * FD_ENTRY_SIZE; this.w64(base + FD_OFFSET, newOffset); @@ -1942,10 +2465,11 @@ export class SharedFS { const entry = this.fdGet(fd); if (!entry) throw new SFSError(EBADF); if ((entry.flags & O_ACCMODE) === O_RDONLY) throw new SFSError(EBADF); + this.validateFileSize(length); this.inodeWriteLock(entry.ino); try { - this.inodeTruncate(entry.ino, length); + this.inodeTruncate(entry.ino, length, true); } finally { this.inodeWriteUnlock(entry.ino); } @@ -1965,6 +2489,10 @@ export class SharedFS { // ── Public API: Path operations ────────────────────────────────── stat(path: string): StatResult { + return this.withNamespaceLock(() => this.statUnlocked(path)); + } + + private statUnlocked(path: string): StatResult { const ino = this.pathResolve(path, true); if (ino < 0) throw new SFSError(ino); this.inodeReadLock(ino); @@ -1976,6 +2504,10 @@ export class SharedFS { } lstat(path: string): StatResult { + return this.withNamespaceLock(() => this.lstatUnlocked(path)); + } + + private lstatUnlocked(path: string): StatResult { const ino = this.pathResolve(path, false); // don't follow symlinks if (ino < 0) throw new SFSError(ino); this.inodeReadLock(ino); @@ -1986,7 +2518,11 @@ export class SharedFS { } } - unlink(path: string): void { + unlink(path: string): NamespaceEntryIdentity { + return this.withNamespaceLock(() => this.unlinkUnlocked(path)); + } + + private unlinkUnlocked(path: string): NamespaceEntryIdentity { const { parentIno, name } = this.pathResolveParent(path); const nameBytes = encoder.encode(name); const requiresDirectory = path.length > 1 && path.endsWith("/"); @@ -2002,6 +2538,7 @@ export class SharedFS { throw new SFSError(ENOTDIR); } if ((mode & S_IFMT) === S_IFDIR) throw new SFSError(EISDIR); + const removed = this.namespaceEntryIdentity(childIno); const rc = this.dirRemoveEntry(parentIno, nameBytes); if (rc < 0) throw new SFSError(rc); @@ -2014,16 +2551,27 @@ export class SharedFS { this.inodeWriteUnlock(childIno); } if (shouldFree) this.inodeFree(childIno); + return removed; } finally { this.inodeWriteUnlock(parentIno); } } - rename(oldPath: string, newPath: string): void { + rename(oldPath: string, newPath: string): RenameIdentityResult { + return this.withNamespaceLock(() => this.renameUnlocked(oldPath, newPath)); + } + + private renameUnlocked( + oldPath: string, + newPath: string, + ): RenameIdentityResult { const { parentIno: oldParent, name: oldName } = this.pathResolveParent(oldPath); const { parentIno: newParent, name: newName } = this.pathResolveParent(newPath); + if (isReservedDirectoryName(oldName) || isReservedDirectoryName(newName)) { + throw new SFSError(EINVAL); + } const oldNameBytes = encoder.encode(oldName); const newNameBytes = encoder.encode(newName); const oldRequiresDirectory = oldPath.length > 1 && oldPath.endsWith("/"); @@ -2041,6 +2589,7 @@ export class SharedFS { const srcOff = this.inodeOffset(srcIno); const srcMode = this.r32(srcOff + INO_MODE); const srcType = srcMode & S_IFMT; + const source = this.namespaceEntryIdentity(srcIno); if ( (oldRequiresDirectory || newRequiresDirectory) && @@ -2053,56 +2602,59 @@ export class SharedFS { throw new SFSError(EINVAL); } - // Remove any existing entry at destination + // Replace an existing destination entry in place. Removing it before + // allocating/inserting the source can destroy the destination when the + // rename later fails with ENOSPC. const existingIno = this.dirLookup(newParent, newNameBytes); let removedExistingDirectory = false; + let replaced: NamespaceEntryIdentity | undefined; if (existingIno >= 0) { if (existingIno === srcIno) { - return; + return { source, replaced: source }; } + replaced = this.namespaceEntryIdentity(existingIno); const existOff = this.inodeOffset(existingIno); const existMode = this.r32(existOff + INO_MODE); const existType = existMode & S_IFMT; - if (srcType === S_IFDIR) { - if (existType !== S_IFDIR) throw new SFSError(ENOTDIR); - - let shouldFreeExisting = false; - this.inodeWriteLock(existingIno); - try { - if (!this.dirIsEmpty(existingIno)) throw new SFSError(ENOTEMPTY); - const rc = this.dirRemoveEntry(newParent, newNameBytes); - if (rc < 0) throw new SFSError(rc); - shouldFreeExisting = this.inodeOrphanLocked(existingIno); - } finally { - this.inodeWriteUnlock(existingIno); - } - if (shouldFreeExisting) this.inodeFree(existingIno); - removedExistingDirectory = true; - } else { - if (existType === S_IFDIR) { - throw new SFSError(newRequiresDirectory ? ENOTDIR : EISDIR); - } + if (srcType === S_IFDIR && existType !== S_IFDIR) { + throw new SFSError(ENOTDIR); + } + if (srcType !== S_IFDIR && existType === S_IFDIR) { + throw new SFSError(EISDIR); + } - const rc = this.dirRemoveEntry(newParent, newNameBytes); - if (rc < 0) throw new SFSError(rc); - let shouldFreeExisting = false; - this.inodeWriteLock(existingIno); - try { - shouldFreeExisting = this.inodeDropLinkRefLocked(existingIno); - } finally { - this.inodeWriteUnlock(existingIno); + let shouldFreeExisting = false; + const existingAlreadyLocked = + existingIno === oldParent || existingIno === newParent; + if (!existingAlreadyLocked) this.inodeWriteLock(existingIno); + try { + if (existType === S_IFDIR && !this.dirIsEmpty(existingIno)) { + throw new SFSError(ENOTEMPTY); } - if (shouldFreeExisting) this.inodeFree(existingIno); + const replaceRc = this.dirReplaceEntryIno( + newParent, + newNameBytes, + srcIno, + ); + if (replaceRc < 0) throw new SFSError(replaceRc); + shouldFreeExisting = + existType === S_IFDIR + ? this.inodeOrphanLocked(existingIno) + : this.inodeDropLinkRefLocked(existingIno); + } finally { + if (!existingAlreadyLocked) this.inodeWriteUnlock(existingIno); } + if (shouldFreeExisting) this.inodeFree(existingIno); + removedExistingDirectory = existType === S_IFDIR; + } else { + const addRc = this.dirAddEntry(newParent, newNameBytes, srcIno); + if (addRc < 0) throw new SFSError(addRc); } - // Add entry in new directory - const rc = this.dirAddEntry(newParent, newNameBytes, srcIno); - if (rc < 0) throw new SFSError(rc); - // Remove entry from old directory - this.dirRemoveEntry(oldParent, oldNameBytes); + const removeRc = this.dirRemoveEntry(oldParent, oldNameBytes); + if (removeRc < 0) throw new SFSError(removeRc); // Update link counts for directory renames if (srcType === S_IFDIR) { @@ -2146,6 +2698,7 @@ export class SharedFS { this.r32(newPOff + INO_LINK_COUNT) - 1, ); } + return { source, replaced }; } finally { if (first !== second) this.inodeWriteUnlock(second); this.inodeWriteUnlock(first); @@ -2153,6 +2706,10 @@ export class SharedFS { } mkdir(path: string, mode: number = 0o755): void { + this.withNamespaceLock(() => this.mkdirUnlocked(path, mode)); + } + + private mkdirUnlocked(path: string, mode: number = 0o755): void { const { parentIno, name } = this.pathResolveParent(path); const nameBytes = encoder.encode(name); @@ -2218,7 +2775,12 @@ export class SharedFS { } rmdir(path: string): void { + this.withNamespaceLock(() => this.rmdirUnlocked(path)); + } + + private rmdirUnlocked(path: string): void { const { parentIno, name } = this.pathResolveParent(path); + if (isReservedDirectoryName(name)) throw new SFSError(EINVAL); const nameBytes = encoder.encode(name); this.inodeWriteLock(parentIno); @@ -2235,7 +2797,8 @@ export class SharedFS { try { if (!this.dirIsEmpty(childIno)) throw new SFSError(ENOTEMPTY); - this.dirRemoveEntry(parentIno, nameBytes); + const removeRc = this.dirRemoveEntry(parentIno, nameBytes); + if (removeRc < 0) throw new SFSError(removeRc); shouldFree = this.inodeOrphanLocked(childIno); } finally { this.inodeWriteUnlock(childIno); @@ -2251,6 +2814,10 @@ export class SharedFS { } symlink(target: string, linkPath: string): void { + this.withNamespaceLock(() => this.symlinkUnlocked(target, linkPath)); + } + + private symlinkUnlocked(target: string, linkPath: string): void { const { parentIno, name } = this.pathResolveParent(linkPath); const nameBytes = encoder.encode(name); const targetBytes = encoder.encode(target); @@ -2279,14 +2846,25 @@ export class SharedFS { targetBytes, targetBytes.length, ); - if (written < 0) { + if (written !== targetBytes.length) { + if (written > 0) this.inodeTruncate(newIno, 0); this.inodeFree(newIno); - throw new SFSError(written); + throw new SFSError(written < 0 ? written : ENOSPC); } } const rc = this.dirAddEntry(parentIno, nameBytes, newIno); if (rc < 0) { + if (targetBytes.length <= INLINE_SYMLINK_SIZE) { + this.u8.fill( + 0, + newOff + INO_DIRECT, + newOff + INO_DIRECT + INLINE_SYMLINK_SIZE, + ); + this.w64(newOff + INO_SIZE, 0); + } else { + this.inodeTruncate(newIno, 0); + } this.inodeFree(newIno); throw new SFSError(rc); } @@ -2296,14 +2874,17 @@ export class SharedFS { } chmod(path: string, mode: number): void { + this.withNamespaceLock(() => this.chmodUnlocked(path, mode)); + } + + private chmodUnlocked(path: string, mode: number): void { const ino = this.pathResolve(path, true); if (ino < 0) throw new SFSError(ino); this.inodeWriteLock(ino); try { const off = this.inodeOffset(ino); const oldMode = this.r32(off + INO_MODE); - const fileType = (mode & S_IFMT) || (oldMode & S_IFMT); - this.w32(off + INO_MODE, fileType | (mode & 0o7777)); + this.w32(off + INO_MODE, (oldMode & S_IFMT) | (mode & 0o7777)); this.w64(off + INO_CTIME, Date.now()); } finally { this.inodeWriteUnlock(ino); @@ -2317,8 +2898,7 @@ export class SharedFS { try { const off = this.inodeOffset(entry.ino); const oldMode = this.r32(off + INO_MODE); - const fileType = (mode & S_IFMT) || (oldMode & S_IFMT); - this.w32(off + INO_MODE, fileType | (mode & 0o7777)); + this.w32(off + INO_MODE, (oldMode & S_IFMT) | (mode & 0o7777)); this.w64(off + INO_CTIME, Date.now()); } finally { this.inodeWriteUnlock(entry.ino); @@ -2326,6 +2906,10 @@ export class SharedFS { } chown(path: string, uid: number, gid: number): void { + this.withNamespaceLock(() => this.chownUnlocked(path, uid, gid)); + } + + private chownUnlocked(path: string, uid: number, gid: number): void { const ino = this.pathResolve(path, true); // POSIX chown follows symlinks if (ino < 0) throw new SFSError(ino); this.inodeWriteLock(ino); @@ -2356,6 +2940,10 @@ export class SharedFS { } lchown(path: string, uid: number, gid: number): void { + this.withNamespaceLock(() => this.lchownUnlocked(path, uid, gid)); + } + + private lchownUnlocked(path: string, uid: number, gid: number): void { const ino = this.pathResolve(path, false); // no-follow: chowns the symlink itself if (ino < 0) throw new SFSError(ino); this.inodeWriteLock(ino); @@ -2369,7 +2957,25 @@ export class SharedFS { } } - utimens(path: string, atimeSec: number, atimeNsec: number, mtimeSec: number, mtimeNsec: number): void { + utimens( + path: string, + atimeSec: number, + atimeNsec: number, + mtimeSec: number, + mtimeNsec: number, + ): void { + this.withNamespaceLock(() => + this.utimensUnlocked(path, atimeSec, atimeNsec, mtimeSec, mtimeNsec), + ); + } + + private utimensUnlocked( + path: string, + atimeSec: number, + atimeNsec: number, + mtimeSec: number, + mtimeNsec: number, + ): void { const ino = this.pathResolve(path, true); if (ino < 0) throw new SFSError(ino); this.inodeWriteLock(ino); @@ -2379,11 +2985,17 @@ export class SharedFS { const UTIME_OMIT = 0x3ffffffe; const now = Date.now(); if (atimeNsec !== UTIME_OMIT) { - const atimeMs = atimeNsec === UTIME_NOW ? now : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); + const atimeMs = + atimeNsec === UTIME_NOW + ? now + : atimeSec * 1000 + Math.floor(atimeNsec / 1_000_000); this.w64(off + INO_ATIME, atimeMs); } if (mtimeNsec !== UTIME_OMIT) { - const mtimeMs = mtimeNsec === UTIME_NOW ? now : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); + const mtimeMs = + mtimeNsec === UTIME_NOW + ? now + : mtimeSec * 1000 + Math.floor(mtimeNsec / 1_000_000); this.w64(off + INO_MTIME, mtimeMs); } this.w64(off + INO_CTIME, now); @@ -2392,8 +3004,19 @@ export class SharedFS { } } - link(existingPath: string, newPath: string): void { - const srcIno = this.pathResolve(existingPath, true); + link(existingPath: string, newPath: string): NamespaceEntryIdentity { + return this.withNamespaceLock(() => + this.linkUnlocked(existingPath, newPath), + ); + } + + private linkUnlocked( + existingPath: string, + newPath: string, + ): NamespaceEntryIdentity { + // Select POSIX link(2)'s permitted no-follow behavior for a final + // symlink, which also matches linkat() when AT_SYMLINK_FOLLOW is clear. + const srcIno = this.pathResolve(existingPath, false); if (srcIno < 0) throw new SFSError(srcIno); const srcOff = this.inodeOffset(srcIno); const srcMode = this.r32(srcOff + INO_MODE); @@ -2416,12 +3039,20 @@ export class SharedFS { } finally { this.inodeWriteUnlock(srcIno); } + return { + ...this.namespaceEntryIdentity(srcIno), + linkCount: this.r32(srcOff + INO_LINK_COUNT), + }; } finally { this.inodeWriteUnlock(parentIno); } } readlink(path: string): string { + return this.withNamespaceLock(() => this.readlinkUnlocked(path)); + } + + private readlinkUnlocked(path: string): string { const ino = this.pathResolve(path, false); if (ino < 0) throw new SFSError(ino); @@ -2449,6 +3080,10 @@ export class SharedFS { // ── Public API: Directory reading ──────────────────────────────── opendir(path: string): number { + return this.withNamespaceLock(() => this.opendirUnlocked(path)); + } + + private opendirUnlocked(path: string): number { const ino = this.pathResolve(path, true); if (ino < 0) throw new SFSError(ino); @@ -2461,7 +3096,11 @@ export class SharedFS { return dd; } - readdirEntry( + readdirEntry(dd: number): { name: string; stat: StatResult } | null { + return this.withNamespaceLock(() => this.readdirEntryUnlocked(dd)); + } + + private readdirEntryUnlocked( dd: number, ): { name: string; stat: StatResult } | null { const entry = this.fdGet(dd); @@ -2475,14 +3114,22 @@ export class SharedFS { const fileBlock = Math.floor(pos / BLOCK_SIZE); const blockOff = pos % BLOCK_SIZE; const phys = this.inodeBlockMap(entry.ino, fileBlock, false); - if (phys <= 0) return null; + if (phys <= 0) throw new SFSError(EIO); const abs = phys * BLOCK_SIZE + blockOff; const entIno = this.r32(abs); const recLen = this.view.getUint16(abs + 4, true); const entNameLen = this.view.getUint16(abs + 6, true); - if (recLen === 0) return null; + if ( + !this.isValidDirEntry( + blockOff, + Math.min(BLOCK_SIZE, blockOff + dirSize - pos), + recLen, + entNameLen, + ) + ) + throw new SFSError(EIO); // Advance offset — update both the SAB (persistent) and the local // snapshot so the while loop progresses past deleted entries (entIno=0). @@ -2491,6 +3138,10 @@ export class SharedFS { this.w64(base + FD_OFFSET, pos + recLen); if (entIno !== 0) { + if (entIno >= this.r32(SB_TOTAL_INODES)) throw new SFSError(EIO); + const ibStart = this.r32(SB_INODE_BITMAP_START) * BLOCK_SIZE; + const word = this.r32(ibStart + (entIno >> 5) * 4); + if ((word & (1 << (entIno & 31))) === 0) throw new SFSError(EIO); const nameStr = safeDecode( this.u8.subarray( abs + DIRENT_HEADER_SIZE, diff --git a/host/test/browser-kernel.test.ts b/host/test/browser-kernel.test.ts index 7107bb115a..a1fc0d3dee 100644 --- a/host/test/browser-kernel.test.ts +++ b/host/test/browser-kernel.test.ts @@ -155,6 +155,93 @@ describe("BrowserKernel", () => { expect(await exit).toBe(7); }); + it("delivers host diagnostics without contaminating guest stderr", async () => { + const BrowserKernel = await loadBrowserKernel(); + const onHostDiagnostic = vi.fn(); + const onStderr = vi.fn(); + const kernel = new BrowserKernel({ + kernelOwnedFs: true, + onHostDiagnostic, + onStderr, + }); + + const bootPromise = kernel.boot({ + kernelWasm: new ArrayBuffer(8), + vfsImage: new Uint8Array(0), + argv: ["/init"], + }); + await new Promise((r) => setTimeout(r, 0)); + const worker = MockWorker.instances[0]!; + worker.simulateMessage({ type: "ready" }); + await new Promise((r) => setTimeout(r, 0)); + const spawn = worker.lastMessage("spawn"); + worker.simulateMessage({ + type: "response", + requestId: spawn.requestId, + result: 100, + }); + await bootPromise; + + worker.simulateMessage({ + type: "host_diagnostic", + pid: 100, + status: 7, + source: "kernel process exit", + message: "[kernel-worker] nonzero process exit pid=100 status=7", + }); + + expect(onHostDiagnostic).toHaveBeenCalledOnce(); + expect(onHostDiagnostic).toHaveBeenCalledWith({ + pid: 100, + status: 7, + source: "kernel process exit", + message: "[kernel-worker] nonzero process exit pid=100 status=7", + }); + expect(onStderr).not.toHaveBeenCalled(); + }); + + it("reports a worker-level error as a host diagnostic, not guest stderr", async () => { + const BrowserKernel = await loadBrowserKernel(); + const onHostDiagnostic = vi.fn(); + const onStderr = vi.fn(); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const kernel = new BrowserKernel({ + kernelOwnedFs: true, + onHostDiagnostic, + onStderr, + }); + + const bootPromise = kernel.boot({ + kernelWasm: new ArrayBuffer(8), + vfsImage: new Uint8Array(0), + argv: ["/init"], + }); + await new Promise((r) => setTimeout(r, 0)); + const worker = MockWorker.instances[0]!; + worker.simulateMessage({ type: "ready" }); + await new Promise((r) => setTimeout(r, 0)); + const spawn = worker.lastMessage("spawn"); + worker.simulateMessage({ + type: "response", + requestId: spawn.requestId, + result: 100, + }); + await bootPromise; + + worker.onerror?.({ message: "worker crashed" }); + + expect(onHostDiagnostic).toHaveBeenCalledOnce(); + expect(onHostDiagnostic).toHaveBeenCalledWith({ + pid: 0, + source: "kernel worker", + message: "[BrowserKernel] kernel worker error: worker crashed", + }); + expect(onStderr).not.toHaveBeenCalled(); + expect(consoleError).toHaveBeenCalledWith( + "[BrowserKernel] kernel worker error: worker crashed", + ); + }); + it("init() waits for lazy VFS registration to be acknowledged", async () => { const { MemoryFileSystem } = await import("../src/vfs/memory-fs"); const BrowserKernel = await loadBrowserKernel(); @@ -180,8 +267,15 @@ describe("BrowserKernel", () => { expect(lazy).toBeDefined(); expect(typeof lazy.requestId).toBe("number"); expect(lazy.entries).toMatchObject([ - { path: "/bin/lazy", url: "/assets/lazy.wasm", size: 123 }, + { + path: "/bin/lazy", + paths: ["/bin/lazy"], + url: "/assets/lazy.wasm", + size: 123, + }, ]); + expect(typeof lazy.entries[0].generation).toBe("number"); + expect(typeof lazy.entries[0].dataSequence).toBe("number"); expect(resolved).toBe(false); w.simulateMessage({ type: "response", requestId: lazy.requestId, result: true }); diff --git a/host/test/fixtures/sharedfs-lock-release-worker.mjs b/host/test/fixtures/sharedfs-lock-release-worker.mjs new file mode 100644 index 0000000000..f190f235fe --- /dev/null +++ b/host/test/fixtures/sharedfs-lock-release-worker.mjs @@ -0,0 +1,10 @@ +import { parentPort, workerData } from "node:worker_threads"; + +const words = new Int32Array(workerData); +const namespaceLockIndex = 64 / Int32Array.BYTES_PER_ELEMENT; +Atomics.store(words, namespaceLockIndex, 1); +parentPort.postMessage("locked"); +setTimeout(() => { + Atomics.store(words, namespaceLockIndex, 0); + Atomics.notify(words, namespaceLockIndex); +}, 100); diff --git a/host/test/fixtures/sharedfs-namespace-worker.ts b/host/test/fixtures/sharedfs-namespace-worker.ts new file mode 100644 index 0000000000..0eb7fd6234 --- /dev/null +++ b/host/test/fixtures/sharedfs-namespace-worker.ts @@ -0,0 +1,57 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { MemoryFileSystem } from "../../src/vfs/memory-fs"; + +const { fsBuffer, controlBuffer, role, iterations } = workerData as { + fsBuffer: SharedArrayBuffer; + controlBuffer: SharedArrayBuffer; + role: "mutator" | "observer"; + iterations: number; +}; +const control = new Int32Array(controlBuffer); +const fs = MemoryFileSystem.fromExisting(fsBuffer); +const O_RDONLY = 0; +const O_WRONLY = 1; +const O_CREAT = 0x40; +const O_TRUNC = 0x200; + +while (Atomics.load(control, 0) === 0) Atomics.wait(control, 0, 0); + +try { + if (role === "mutator") { + for (let i = 0; i < iterations; i++) { + try { fs.unlink("/slot"); } catch { /* observer may hold an unlinked fd */ } + + let fd = fs.open("/other", O_WRONLY | O_CREAT | O_TRUNC, 0o644); + fs.write(fd, new Uint8Array([0x22]), null, 1); + fs.close(fd); + fs.unlink("/other"); + + fd = fs.open("/slot", O_WRONLY | O_CREAT | O_TRUNC, 0o644); + fs.write(fd, new Uint8Array([0x11]), null, 1); + fs.close(fd); + } + } else { + for (let i = 0; i < iterations; i++) { + let fd: number | null = null; + try { + fd = fs.open("/slot", O_RDONLY, 0); + const byte = new Uint8Array(1); + const read = fs.read(fd, byte, null, 1); + if (read === 1 && byte[0] !== 0x11) { + throw new Error(`path ABA exposed data from recycled inode: ${byte[0]}`); + } + } catch (error) { + if (error instanceof Error && error.message.includes("path ABA")) throw error; + // ENOENT is valid between unlink and recreate. + } finally { + if (fd !== null) fs.close(fd); + } + } + } + parentPort!.postMessage({ ok: true }); +} catch (error) { + parentPort!.postMessage({ + ok: false, + error: error instanceof Error ? error.stack ?? error.message : String(error), + }); +} diff --git a/host/test/lazy-archive.test.ts b/host/test/lazy-archive.test.ts index 5e18ba65a3..218860ac81 100644 --- a/host/test/lazy-archive.test.ts +++ b/host/test/lazy-archive.test.ts @@ -5,6 +5,8 @@ import { parseZipCentralDirectory } from "../src/vfs/zip"; import type { ZipEntry } from "../src/vfs/zip"; const O_RDONLY = 0x0000; +const O_WRONLY = 0x0001; +const O_TRUNC = 0x0200; function createMemfs(): MemoryFileSystem { const sab = new SharedArrayBuffer(4 * 1024 * 1024); @@ -61,9 +63,170 @@ function makeRealZip() { return { zipBytes, entries: parseZipCentralDirectory(zipBytes) }; } +function makeTwoMemberZip() { + const zipBytes = zipSync({ + "a.txt": new TextEncoder().encode("alpha"), + "b.txt": new TextEncoder().encode("bravo"), + }); + return { zipBytes, entries: parseZipCentralDirectory(zipBytes) }; +} + +function readText(mfs: MemoryFileSystem, path: string): string { + const fd = mfs.open(path, O_RDONLY, 0); + const buffer = new Uint8Array(64); + const read = mfs.read(fd, buffer, null, buffer.length); + mfs.close(fd); + return new TextDecoder().decode(buffer.subarray(0, read)); +} + // --- Task 3: Registration --- describe("Lazy archive group registration", () => { + it("atomically replaces an existing file with archive backing", async () => { + const originalFetch = globalThis.fetch; + const mfs = createMemfs(); + const { zipBytes, entries } = makeRealZip(); + mfs.mkdir("/opt", 0o755); + mfs.mkdir("/opt/bin", 0o755); + mfs.createFileWithOwner("/opt/bin/hello", 0o644, 0, 0, new Uint8Array([9])); + + mfs.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + try { + await expect(mfs.ensureMaterialized("/opt/bin/hello")).resolves.toBe( + true, + ); + expect(readText(mfs, "/opt/bin/hello")).toBe("#!/bin/sh\necho hello"); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("does not overwrite a peer write after replacing an archive path", async () => { + const originalFetch = globalThis.fetch; + const mfs = createMemfs(); + const peer = MemoryFileSystem.fromExisting(mfs.sharedBuffer); + const { zipBytes, entries } = makeRealZip(); + mfs.mkdir("/opt", 0o755); + mfs.mkdir("/opt/bin", 0o755); + mfs.createFileWithOwner("/opt/bin/hello", 0o644, 0, 0, new Uint8Array([4])); + + const raw = ( + mfs as unknown as { + fs: { + createLazyStub: ( + path: string, + mode: number, + ) => { ino: number; generation: number; dataSequence: number }; + }; + } + ).fs; + const createLazyStub = raw.createLazyStub.bind(raw); + const createSpy = vi + .spyOn(raw, "createLazyStub") + .mockImplementation((path, mode) => { + const identity = createLazyStub(path, mode); + if (path === "/opt/bin/hello") { + const writer = peer.open(path, O_WRONLY, 0o644); + peer.write(writer, new Uint8Array([9]), null, 1); + peer.close(writer); + } + return identity; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + try { + mfs.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + await expect(mfs.ensureMaterialized("/opt/bin/hello")).resolves.toBe( + true, + ); + const fd = mfs.open("/opt/bin/hello", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(mfs.read(fd, byte, null, 1)).toBe(1); + mfs.close(fd); + expect(byte[0]).toBe(9); + } finally { + createSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("replaces stale standalone backing when a path moves into an archive", async () => { + const originalFetch = globalThis.fetch; + const mfs = createMemfs(); + const zipBytes = zipSync({ item: new Uint8Array([7]) }); + mfs.registerLazyFile("/item", "http://example.com/old", 1); + mfs.registerLazyArchiveFromEntries( + "http://example.com/archive.zip", + parseZipCentralDirectory(zipBytes), + "/", + ); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + globalThis.fetch = fetchMock; + + try { + await expect(mfs.ensureMaterialized("/item")).resolves.toBe(true); + expect(fetchMock).toHaveBeenCalledWith("http://example.com/archive.zip"); + const fd = mfs.open("/item", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(mfs.read(fd, byte, null, 1)).toBe(1); + mfs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("replaces stale archive backing when a member becomes standalone", async () => { + const originalFetch = globalThis.fetch; + const mfs = createMemfs(); + const zipBytes = zipSync({ item: new Uint8Array([7]) }); + mfs.registerLazyArchiveFromEntries( + "http://example.com/archive.zip", + parseZipCentralDirectory(zipBytes), + "/", + ); + mfs.registerLazyFile("/item", "http://example.com/standalone", 1); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([8]).buffer), + } as unknown as Response); + globalThis.fetch = fetchMock; + + try { + expect(mfs.exportLazyArchiveEntries()).toEqual([]); + await expect(mfs.ensureMaterialized("/item")).resolves.toBe(true); + expect(fetchMock).toHaveBeenCalledWith("http://example.com/standalone"); + const fd = mfs.open("/item", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(mfs.read(fd, byte, null, 1)).toBe(1); + mfs.close(fd); + expect(byte[0]).toBe(8); + } finally { + globalThis.fetch = originalFetch; + } + }); + it("registerLazyArchiveFromEntries creates stubs for all files", () => { const mfs = createMemfs(); const entries = makeFakeEntries(); @@ -404,6 +567,85 @@ describe("Lazy archive materialization", () => { const st = mfs.stat("/opt/bin/hello"); expect(st.mode & 0o777).toBe(0o700); }); + + it("keeps a peer-renamed member lazy when another archive member materializes", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const peer = MemoryFileSystem.fromExisting(sab); + const { zipBytes, entries } = makeTwoMemberZip(); + + owner.registerLazyArchiveFromEntries( + "http://example.com/two-members.zip", + entries, + "/pkg", + ); + peer.rename("/pkg/b.txt", "/pkg/moved-b.txt"); + + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + await expect(owner.ensureMaterialized("/pkg/a.txt")).resolves.toBe(true); + expect(readText(owner, "/pkg/a.txt")).toBe("alpha"); + + await expect(owner.ensureMaterialized("/pkg/moved-b.txt")).resolves.toBe( + true, + ); + expect(readText(owner, "/pkg/moved-b.txt")).toBe("bravo"); + }); + + it("finishes a requested archive member renamed during its fetch", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const peer = MemoryFileSystem.fromExisting(sab); + const { zipBytes, entries } = makeTwoMemberZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/two-members.zip", + entries, + "/pkg", + ); + let release!: (value: ArrayBuffer) => void; + const body = new Promise((resolve) => { + release = resolve; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => body, + } as unknown as Response); + + const pending = owner.ensureMaterialized("/pkg/a.txt"); + await vi.waitFor(() => expect(globalThis.fetch).toHaveBeenCalled()); + peer.rename("/pkg/a.txt", "/pkg/moved-a.txt"); + release(zipBytes.buffer); + + await expect(pending).resolves.toBe(true); + expect(readText(owner, "/pkg/moved-a.txt")).toBe("alpha"); + }); + + it("materializes pending archives into a self-contained image", async () => { + const mfs = createMemfs(); + const { zipBytes, entries } = makeRealZip(); + mfs.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + globalThis.fetch = fetchMock; + + const restored = MemoryFileSystem.fromImage( + await mfs.saveImage({ materializeAll: true }), + ); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(restored.exportLazyArchiveEntries()).toEqual([]); + expect(readText(restored, "/opt/bin/hello")).toBe("#!/bin/sh\necho hello"); + expect(readText(restored, "/opt/share/data.txt")).toBe("hello world"); + }); }); // --- Task 5: Unlink tracking --- @@ -467,6 +709,60 @@ describe("Lazy archive unlink tracking", () => { expect(new TextDecoder().decode(buf.subarray(0, n))).toBe("hello world"); }); + it("retains a peer-created hard-link alias after unlink", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const peer = MemoryFileSystem.fromExisting(sab); + const { zipBytes, entries } = makeRealZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + peer.link("/opt/bin/hello", "/archive-alias"); + + owner.unlink("/opt/bin/hello"); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + await expect(owner.ensureMaterialized("/archive-alias")).resolves.toBe( + true, + ); + expect(readText(owner, "/archive-alias")).toBe("#!/bin/sh\necho hello"); + }); + + it("retains a peer-created archive alias when rename replaces its other name", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const peer = MemoryFileSystem.fromExisting(sab); + const { zipBytes, entries } = makeRealZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + peer.link("/opt/bin/hello", "/archive-alias"); + owner.createFileWithOwner("/source", 0o644, 0, 0, new Uint8Array([4])); + + owner.rename("/source", "/opt/bin/hello"); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + await expect(owner.ensureMaterialized("/archive-alias")).resolves.toBe( + true, + ); + expect(readText(owner, "/archive-alias")).toBe("#!/bin/sh\necho hello"); + const destination = owner.open("/opt/bin/hello", O_RDONLY, 0); + const destinationByte = new Uint8Array(1); + expect(owner.read(destination, destinationByte, null, 1)).toBe(1); + owner.close(destination); + expect(destinationByte[0]).toBe(4); + }); + it("unlink of non-archive file does not affect archive groups", () => { const mfs = createMemfs(); const entries = makeFakeEntries(); @@ -530,6 +826,65 @@ describe("Lazy archive export/import", () => { expect(mfs2.stat("/usr/share/vim/README").size).toBe(0); }); + it("retains the original archive member name when importing legacy metadata", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const { zipBytes, entries } = makeRealZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + + const serialized = owner.exportLazyArchiveEntries(); + for (const entry of serialized[0].entries) delete entry.archivePath; + + const restoredMetadata = MemoryFileSystem.fromExisting(sab); + restoredMetadata.importLazyArchiveEntries(serialized); + restoredMetadata.rename("/opt/bin/hello", "/opt/bin/moved"); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + arrayBuffer: () => Promise.resolve(zipBytes.buffer), + } as unknown as Response); + + await expect( + restoredMetadata.ensureMaterialized("/opt/bin/moved"), + ).resolves.toBe(true); + expect(readText(restoredMetadata, "/opt/bin/moved")).toBe( + "#!/bin/sh\necho hello", + ); + }); + + it("rejects sequence-less archive metadata after a live peer write", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const owner = MemoryFileSystem.create(sab); + const { entries } = makeRealZip(); + owner.registerLazyArchiveFromEntries( + "http://example.com/test.zip", + entries, + "/opt", + ); + const serialized = owner.exportLazyArchiveEntries(); + const hello = serialized[0].entries.find( + (entry) => entry.vfsPath === "/opt/bin/hello", + )!; + delete hello.dataSequence; + + const writer = owner.open("/opt/bin/hello", O_WRONLY | O_TRUNC, 0o644); + owner.write(writer, new Uint8Array([9]), null, 1); + owner.close(writer); + + const peer = MemoryFileSystem.fromExisting(sab); + expect(() => peer.importLazyArchiveEntries(serialized)).toThrow( + /requires inode generation and data sequence/, + ); + + expect(peer.stat("/opt/bin/hello").size).toBe(1); + await expect(peer.ensureMaterialized("/opt/bin/hello")).resolves.toBe( + false, + ); + }); + it("rebaseToNewFileSystem preserves unmaterialized archive metadata", () => { const mfs = createMemfs(); const entries = makeFakeEntries(); @@ -557,7 +912,7 @@ describe("Lazy archive export/import", () => { rebased.close(fd); }); - it("import of materialized group does not populate lazyArchiveInodes", async () => { + it("omits fully materialized groups from deferred archive metadata", async () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs1 = MemoryFileSystem.create(sab); const { zipBytes, entries } = makeRealZip(); @@ -576,10 +931,9 @@ describe("Lazy archive export/import", () => { await mfs1.ensureMaterialized("/opt/bin/hello"); - // Export from instance 1 — group should be materialized + // Concrete files no longer need archive metadata in another instance. const serialized = mfs1.exportLazyArchiveEntries(); - expect(serialized).toHaveLength(1); - expect(serialized[0].materialized).toBe(true); + expect(serialized).toEqual([]); // Import into instance 2 on the same SAB const mfs2 = MemoryFileSystem.fromExisting(sab); diff --git a/host/test/sharedfs-safety.test.ts b/host/test/sharedfs-safety.test.ts new file mode 100644 index 0000000000..557e704415 --- /dev/null +++ b/host/test/sharedfs-safety.test.ts @@ -0,0 +1,937 @@ +import { describe, expect, it, vi } from "vitest"; +import { Worker } from "node:worker_threads"; +import { MemoryFileSystem } from "../src/vfs/memory-fs"; + +const O_RDONLY = 0x0000; +const O_WRONLY = 0x0001; +const O_RDWR = 0x0002; +const O_CREAT = 0x0040; +const O_TRUNC = 0x0200; +const SEEK_SET = 0; + +function create(size = 4 * 1024 * 1024): MemoryFileSystem { + return MemoryFileSystem.create(new SharedArrayBuffer(size)); +} + +function listDir(fs: MemoryFileSystem, path: string): string[] { + const dd = fs.opendir(path); + const names: string[] = []; + try { + for (;;) { + const entry = fs.readdir(dd); + if (!entry) return names; + if (entry.name !== "." && entry.name !== "..") names.push(entry.name); + } + } finally { + fs.closedir(dd); + } +} + +describe("SharedFS sparse-file safety", () => { + it("extends a multi-gigabyte sparse file without scanning its holes", () => { + const fs = create(); + const fd = fs.open("/sparse", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const sparseSize = 3_000_000_000; + + const started = performance.now(); + fs.ftruncate(fd, sparseSize); + expect(performance.now() - started).toBeLessThan(500); + expect(fs.fstat(fd).size).toBe(sparseSize); + + fs.seek(fd, sparseSize - 1, SEEK_SET); + expect(fs.write(fd, new Uint8Array([0x7a]), null, 1)).toBe(1); + const tail = new Uint8Array(4); + expect(fs.read(fd, tail, sparseSize - 3, tail.length)).toBe(3); + expect(Array.from(tail.subarray(0, 3))).toEqual([0, 0, 0x7a]); + fs.close(fd); + }); + + it("rejects invalid and unrepresentable truncate lengths", () => { + const fs = create(); + const fd = fs.open("/file", O_CREAT | O_RDWR, 0o644); + expect(() => fs.ftruncate(fd, -1)).toThrow(/Invalid argument/); + expect(() => fs.ftruncate(fd, Number.NaN)).toThrow(/Invalid argument/); + expect(() => fs.ftruncate(fd, Number.MAX_SAFE_INTEGER)).toThrow( + /File too large/, + ); + expect(fs.fstat(fd).size).toBe(0); + fs.close(fd); + }); + + it("commits size and data for a positive partial write at ENOSPC", () => { + const fs = create(128 * 1024); + const fd = fs.open("/partial", O_CREAT | O_RDWR | O_TRUNC, 0o644); + const input = new Uint8Array(1024 * 1024).fill(0xa5); + const written = fs.write(fd, input, null, input.length); + + expect(written).toBeGreaterThan(0); + expect(written).toBeLessThan(input.length); + expect(fs.fstat(fd).size).toBe(written); + const tail = new Uint8Array(1); + expect(fs.read(fd, tail, written - 1, 1)).toBe(1); + expect(tail[0]).toBe(0xa5); + fs.close(fd); + }); +}); + +describe("SharedFS namespace and image safety", () => { + it("hard-links a symlink inode without following its target", () => { + const fs = create(); + const target = fs.open("/target", O_CREAT | O_WRONLY, 0o644); + fs.close(target); + fs.symlink("/target", "/symbolic"); + + fs.link("/symbolic", "/alias"); + expect(fs.lstat("/alias").mode & 0xf000).toBe(0xa000); + expect(fs.lstat("/alias").nlink).toBe(2); + expect(fs.readlink("/alias")).toBe("/target"); + + fs.unlink("/symbolic"); + expect(fs.lstat("/alias").mode & 0xf000).toBe(0xa000); + expect(fs.lstat("/alias").nlink).toBe(1); + }); + + it("prevents path ABA across workers without deadlocking", async () => { + const fs = create(); + const initial = fs.open("/slot", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(initial, new Uint8Array([0x11]), null, 1); + fs.close(initial); + + const controlBuffer = new SharedArrayBuffer(4); + const control = new Int32Array(controlBuffer); + const workerUrl = new URL( + "./fixtures/sharedfs-namespace-worker.ts", + import.meta.url, + ); + const makeWorker = (role: "mutator" | "observer") => + new Worker(workerUrl, { + execArgv: ["--import", "tsx"], + workerData: { + fsBuffer: fs.sharedBuffer, + controlBuffer, + role, + iterations: 4_000, + }, + }); + const workers = [makeWorker("mutator"), makeWorker("observer")]; + const results = workers.map( + (worker) => + new Promise<{ ok: boolean; error?: string }>((resolve, reject) => { + worker.once("message", resolve); + worker.once("error", reject); + worker.once("exit", (code) => { + if (code !== 0) reject(new Error(`SharedFS worker exited ${code}`)); + }); + }), + ); + + Atomics.store(control, 0, 1); + Atomics.notify(control, 0, workers.length); + try { + const completed = Promise.all(results); + const timeout = new Promise((_, reject) => { + setTimeout( + () => reject(new Error("SharedFS worker watchdog expired")), + 8_000, + ); + }); + const messages = await Promise.race([completed, timeout]); + expect(messages).toEqual([{ ok: true }, { ok: true }]); + } finally { + await Promise.all(workers.map((worker) => worker.terminate())); + } + }, 10_000); + + it("polls safely when browser-main-style Atomics.wait is unavailable", async () => { + const fs = create(); + const worker = new Worker( + new URL("./fixtures/sharedfs-lock-release-worker.mjs", import.meta.url), + { workerData: fs.sharedBuffer }, + ); + await new Promise((resolve, reject) => { + worker.once("message", () => resolve()); + worker.once("error", reject); + }); + + const wait = vi.spyOn(Atomics, "wait").mockImplementation(() => { + throw new TypeError("Atomics.wait cannot be called on this thread"); + }); + try { + expect(fs.stat("/").mode & 0xf000).toBe(0x4000); + expect(wait).toHaveBeenCalled(); + } finally { + wait.mockRestore(); + await worker.terminate(); + } + }); + + it("leaves a full directory intact after repeated create and rename failures", () => { + const fs = create(128 * 1024); + const filler = fs.open("/filler", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(filler, new Uint8Array(1024 * 1024), null, 1024 * 1024); + fs.close(filler); + + const created: string[] = []; + let failedName = ""; + for (let i = 0; i < 64; i++) { + const name = `/${String(i).padStart(2, "0")}-${"x".repeat(180)}`; + try { + const fd = fs.open(name, O_CREAT | O_WRONLY, 0o644); + fs.close(fd); + created.push(name); + } catch (error) { + expect(String(error)).toMatch(/No space left/); + failedName = name; + break; + } + } + + expect(failedName).not.toBe(""); + const before = listDir(fs, "/").sort(); + for (let attempt = 0; attempt < 3; attempt++) { + expect(() => fs.open(failedName, O_CREAT | O_WRONLY, 0o644)).toThrow( + /No space left/, + ); + expect(listDir(fs, "/").sort()).toEqual(before); + } + + const source = created[0]; + const destination = `/${"r".repeat(220)}`; + expect(() => fs.rename(source, destination)).toThrow(/No space left/); + expect(fs.stat(source).mode & 0xf000).toBe(0x8000); + expect(() => fs.stat(destination)).toThrow(/No such file/); + expect(listDir(fs, "/").sort()).toEqual(before); + }); + + it("requires quiescent snapshots and clears legacy runtime state on restore", async () => { + const fs = create(); + const fd = fs.open("/saved", O_CREAT | O_RDWR, 0o644); + await expect(fs.saveImage()).rejects.toThrow(/open descriptors/); + fs.close(fd); + const orphanFd = fs.open("/orphan", O_CREAT | O_RDWR, 0o644); + const orphanIno = fs.fstat(orphanFd).ino; + fs.close(orphanFd); + + const image = await fs.saveImage(); + const imageView = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); + const sabOffset = 16; + imageView.setUint32(sabOffset + 60, 1, true); // stale grow lock + imageView.setUint32(sabOffset + 64, 1, true); // stale namespace lock + imageView.setUint32(sabOffset + 256, 1, true); // stale fd 0 + + const inodeTableBlock = imageView.getUint32(sabOffset + 36, true); + const ino = fs.stat("/saved").ino; + const inodeOffset = sabOffset + inodeTableBlock * 4096 + (ino % 32) * 128; + imageView.setUint32(inodeOffset, 0x80000000, true); // stale inode lock + imageView.setUint32(inodeOffset + 112, 1, true); // stale open ref + + const orphanOffset = + sabOffset + inodeTableBlock * 4096 + (orphanIno % 32) * 128; + imageView.setUint32(orphanOffset + 12, 0, true); // unlinked + imageView.setUint32(orphanOffset + 112, 1, true); // but held open + imageView.setUint32(sabOffset + 256 + 4, orphanIno, true); + + const rootOffset = sabOffset + inodeTableBlock * 4096 + 128; + const rootBlock = imageView.getUint32(rootOffset + 48, true); + const rootSize = Number(imageView.getBigUint64(rootOffset + 16, true)); + const decoder = new TextDecoder(); + for (let pos = 0; pos < rootSize;) { + const entryOffset = sabOffset + rootBlock * 4096 + pos; + const recLen = imageView.getUint16(entryOffset + 4, true); + const nameLen = imageView.getUint16(entryOffset + 6, true); + const name = decoder.decode( + image.subarray(entryOffset + 8, entryOffset + 8 + nameLen), + ); + if (name === "orphan") imageView.setUint32(entryOffset, 0, true); + pos += recLen; + } + + const restored = MemoryFileSystem.fromImage(image); + expect(restored.stat("/saved").ino).toBe(ino); + expect(() => restored.stat("/orphan")).toThrow(/No such file/); + const replacement = restored.open( + "/replacement", + O_CREAT | O_WRONLY, + 0o644, + ); + expect(restored.fstat(replacement).ino).toBe(orphanIno); + restored.close(replacement); + expect(() => restored.close(0)).toThrow(/Bad file descriptor/); + await expect(restored.saveImage()).resolves.toBeInstanceOf(Uint8Array); + }); + + it("fails closed instead of removing a corrupt directory", () => { + const fs = create(); + fs.mkdir("/empty", 0o755); + const ino = fs.stat("/empty").ino; + const view = new DataView(fs.sharedBuffer); + const inodeTableBlock = view.getUint32(36, true); + const inodeOffset = inodeTableBlock * 4096 + (ino % 32) * 128; + const dataBlock = view.getUint32(inodeOffset + 48, true); + const recLenOffset = dataBlock * 4096 + 4; + const originalRecLen = view.getUint16(recLenOffset, true); + + view.setUint16(recLenOffset, 0, true); + expect(() => fs.rmdir("/empty")).toThrow(/I\/O error/); + expect(fs.stat("/empty").ino).toBe(ino); + + view.setUint16(recLenOffset, originalRecLen, true); + fs.rmdir("/empty"); + expect(() => fs.stat("/empty")).toThrow(/No such file/); + }); + + it("rejects a directory entry that names a free inode slot", () => { + const fs = create(); + const fd = fs.open("/victim", O_CREAT | O_WRONLY, 0o644); + fs.close(fd); + const view = new DataView(fs.sharedBuffer); + const inodeTableBlock = view.getUint32(36, true); + const rootOffset = inodeTableBlock * 4096 + 128; + const rootBlock = view.getUint32(rootOffset + 48, true); + const rootSize = Number(view.getBigUint64(rootOffset + 16, true)); + const decoder = new TextDecoder(); + let victimEntry = -1; + for (let pos = 0; pos < rootSize;) { + const abs = rootBlock * 4096 + pos; + const recLen = view.getUint16(abs + 4, true); + const nameLen = view.getUint16(abs + 6, true); + const name = decoder.decode( + new Uint8Array(fs.sharedBuffer, abs + 8, nameLen), + ); + if (name === "victim") victimEntry = abs; + pos += recLen; + } + expect(victimEntry).toBeGreaterThan(0); + + const freeInode = view.getUint32(16, true) - 1; + view.setUint32(victimEntry, freeInode, true); + const freeBefore = fs.statfs("/").ffree; + expect(() => fs.stat("/victim")).toThrow(/I\/O error/); + expect(() => fs.unlink("/victim")).toThrow(/I\/O error/); + expect(fs.statfs("/").ffree).toBe(freeBefore); + }); +}); + +describe("MemoryFileSystem lazy inode identity", () => { + it("atomically replaces an existing file with lazy backing", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const existing = fs.open("/lazy", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(existing, new Uint8Array([9]), null, 1); + fs.close(existing); + + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + expect(fs.getLazyEntry("/lazy")).toMatchObject({ size: 1 }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/lazy")).resolves.toBe(true); + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("does not overwrite a peer write after replacing an existing lazy path", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + const existing = fs.open("/lazy", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(existing, new Uint8Array([4]), null, 1); + fs.close(existing); + + const raw = ( + fs as unknown as { + fs: { + createLazyStub: ( + path: string, + mode: number, + ) => { ino: number; generation: number; dataSequence: number }; + }; + } + ).fs; + const createLazyStub = raw.createLazyStub.bind(raw); + const createSpy = vi + .spyOn(raw, "createLazyStub") + .mockImplementation((path, mode) => { + const identity = createLazyStub(path, mode); + const writer = peer.open(path, O_WRONLY, 0o644); + peer.write(writer, new Uint8Array([9]), null, 1); + peer.close(writer); + return identity; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + expect(fs.getLazyEntry("/lazy")).toBeNull(); + await expect(fs.ensureMaterialized("/lazy")).resolves.toBe(false); + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(9); + } finally { + createSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("binds registration to the atomically-created stub identity", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + type StubIdentity = ReturnType<(typeof fs)["registerLazyFile"]>; + const raw = ( + fs as unknown as { + fs: { + createLazyStub: ( + path: string, + mode: number, + ) => { + ino: StubIdentity; + generation: number; + dataSequence: number; + }; + }; + } + ).fs; + const createLazyStub = raw.createLazyStub.bind(raw); + const createSpy = vi + .spyOn(raw, "createLazyStub") + .mockImplementation((path, mode) => { + const identity = createLazyStub(path, mode); + peer.rename(path, "/moved"); + const replacement = peer.open(path, O_CREAT | O_WRONLY, 0o644); + peer.write(replacement, new Uint8Array([9]), null, 1); + peer.close(replacement); + return identity; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + expect(fs.getLazyEntry("/lazy")).toBeNull(); + expect(fs.getLazyEntry("/moved")).not.toBeNull(); + await expect(fs.ensureMaterialized("/moved")).resolves.toBe(true); + const replacement = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(replacement, byte, null, 1)).toBe(1); + fs.close(replacement); + expect(byte[0]).toBe(9); + } finally { + createSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("rejects generation-less lazy metadata from a live peer", () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 999); + const [legacy] = fs.exportLazyEntries(); + delete legacy.generation; + delete legacy.dataSequence; + fs.unlink("/lazy"); + const replacement = fs.open("/lazy", O_CREAT | O_WRONLY, 0o644); + fs.close(replacement); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + expect(() => peer.importLazyEntries([legacy])).toThrow( + /requires inode generation and data sequence/, + ); + + expect(peer.getLazyEntry("/lazy")).toBeNull(); + expect(peer.stat("/lazy").size).toBe(0); + }); + + it("rejects sequence-less lazy metadata after same-inode content changes", () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 999); + const [legacy] = fs.exportLazyEntries(); + delete legacy.dataSequence; + + const writer = fs.open("/lazy", O_WRONLY | O_TRUNC, 0o644); + fs.write(writer, new Uint8Array([9]), null, 1); + fs.close(writer); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + expect(() => peer.importLazyEntries([legacy])).toThrow( + /requires inode generation and data sequence/, + ); + + expect(peer.getLazyEntry("/lazy")).toBeNull(); + expect(peer.stat("/lazy").size).toBe(1); + }); + + it("does not apply lazy metadata after an inode slot is recycled", () => { + const fs = create(); + const oldIno = fs.registerLazyFile( + "/lazy", + "https://example.test/lazy", + 999, + ); + const exported = fs.exportLazyEntries(); + fs.unlink("/lazy"); + + const fd = fs.open("/replacement", O_CREAT | O_WRONLY, 0o644); + fs.close(fd); + expect(fs.stat("/replacement").ino).toBe(oldIno); + expect(fs.stat("/replacement").size).toBe(0); + expect(fs.getLazyEntry("/replacement")).toBeNull(); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + peer.importLazyEntries(exported); + expect(peer.stat("/replacement").size).toBe(0); + expect(peer.getLazyEntry("/replacement")).toBeNull(); + }); + + it("tracks lazy files across rename and hard-link lifecycle", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 3); + fs.rename("/lazy", "/renamed"); + fs.link("/renamed", "/alias"); + fs.unlink("/renamed"); + + expect(fs.getLazyEntry("/alias")).toMatchObject({ + path: "/alias", + size: 3, + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "3" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([1, 2, 3]).buffer), + } as unknown as Response); + try { + await expect(fs.ensureMaterialized("/alias")).resolves.toBe(true); + } finally { + globalThis.fetch = originalFetch; + } + expect(fs.stat("/alias").size).toBe(3); + }); + + it("retains a peer-created hard-link alias after unlink", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + peer.link("/lazy", "/alias"); + + fs.unlink("/lazy"); + expect(fs.getLazyEntry("/alias")).toMatchObject({ + path: "/alias", + paths: ["/alias"], + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/alias")).resolves.toBe(true); + const fd = fs.open("/alias", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("retains a peer-created lazy alias when rename replaces its other name", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + fs.registerLazyFile("/destination", "https://example.test/lazy", 1); + peer.link("/destination", "/alias"); + const source = fs.open("/source", O_CREAT | O_WRONLY | O_TRUNC, 0o644); + fs.write(source, new Uint8Array([4]), null, 1); + fs.close(source); + + fs.rename("/source", "/destination"); + expect(fs.getLazyEntry("/alias")).toMatchObject({ path: "/alias" }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/alias")).resolves.toBe(true); + const alias = fs.open("/alias", O_RDONLY, 0); + const aliasByte = new Uint8Array(1); + expect(fs.read(alias, aliasByte, null, 1)).toBe(1); + fs.close(alias); + expect(aliasByte[0]).toBe(7); + + const destination = fs.open("/destination", O_RDONLY, 0); + const destinationByte = new Uint8Array(1); + expect(fs.read(destination, destinationByte, null, 1)).toBe(1); + fs.close(destination); + expect(destinationByte[0]).toBe(4); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("uses the identity actually removed when unlink races a peer rename", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + const raw = ( + fs as unknown as { + fs: { unlink: (path: string) => unknown }; + } + ).fs; + const unlink = raw.unlink.bind(raw); + const unlinkSpy = vi.spyOn(raw, "unlink").mockImplementation((path) => { + peer.rename(path, "/moved"); + const replacement = peer.open(path, O_CREAT | O_WRONLY, 0o644); + peer.write(replacement, new Uint8Array([9]), null, 1); + peer.close(replacement); + return unlink(path); + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + fs.unlink("/lazy"); + expect(fs.getLazyEntry("/moved")).not.toBeNull(); + await expect(fs.ensureMaterialized("/moved")).resolves.toBe(true); + } finally { + unlinkSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("uses the source actually renamed when a peer replaces the old path", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + const raw = ( + fs as unknown as { + fs: { rename: (oldPath: string, newPath: string) => unknown }; + } + ).fs; + const rename = raw.rename.bind(raw); + const renameSpy = vi + .spyOn(raw, "rename") + .mockImplementation((oldPath, newPath) => { + peer.rename(oldPath, "/moved"); + const replacement = peer.open(oldPath, O_CREAT | O_WRONLY, 0o644); + peer.write(replacement, new Uint8Array([9]), null, 1); + peer.close(replacement); + return rename(oldPath, newPath); + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + fs.rename("/lazy", "/renamed"); + expect(fs.getLazyEntry("/moved")).not.toBeNull(); + expect(fs.stat("/renamed").size).toBe(1); + await expect(fs.ensureMaterialized("/moved")).resolves.toBe(true); + } finally { + renameSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("preserves every lazy hard-link name across instance transfer", () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 3); + fs.link("/lazy", "/alias"); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + peer.importLazyEntries(fs.exportLazyEntries()); + peer.unlink("/lazy"); + + expect(peer.getLazyEntry("/alias")).toMatchObject({ + path: "/alias", + paths: ["/alias"], + size: 3, + }); + }); + + it("preserves ordinary and lazy hard-link identity while rebasing", () => { + const fs = create(); + const regular = fs.open("/regular", O_CREAT | O_WRONLY, 0o644); + fs.write(regular, new Uint8Array([1, 2, 3]), null, 3); + fs.close(regular); + fs.link("/regular", "/regular-alias"); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 3); + fs.link("/lazy", "/lazy-alias"); + + const rebased = fs.rebaseToNewFileSystem(8 * 1024 * 1024); + expect(rebased.stat("/regular").ino).toBe( + rebased.stat("/regular-alias").ino, + ); + expect(rebased.stat("/regular").nlink).toBe(2); + expect(rebased.stat("/lazy").ino).toBe(rebased.stat("/lazy-alias").ino); + expect(rebased.stat("/lazy").nlink).toBe(2); + expect(rebased.getLazyEntry("/lazy-alias")).toMatchObject({ + size: 3, + paths: expect.arrayContaining(["/lazy", "/lazy-alias"]), + }); + }); + + it("rebases from one coherent snapshot when a peer renames afterward", () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 3); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + + // Inject the peer mutation immediately after SharedFS captures its bytes + // and identity map. The rebased result must consistently reflect that + // captured pre-rename state rather than walking the newer live tree. + const shared = ( + fs as unknown as { + fs: { snapshotState: () => unknown }; + } + ).fs; + const snapshotState = shared.snapshotState.bind(shared); + shared.snapshotState = () => { + const snapshot = snapshotState(); + peer.rename("/lazy", "/moved"); + return snapshot; + }; + + const rebased = fs.rebaseToNewFileSystem(8 * 1024 * 1024); + expect(peer.stat("/moved").size).toBe(0); + expect(() => peer.stat("/lazy")).toThrow(); + expect(rebased.stat("/lazy").size).toBe(3); + expect(() => rebased.stat("/moved")).toThrow(); + expect(rebased.getLazyEntry("/lazy")).toMatchObject({ + path: "/lazy", + size: 3, + }); + }); + + it("drops deferred backing after an explicit write through a hard link", async () => { + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 999); + fs.link("/lazy", "/alias"); + + const fd = fs.open("/alias", O_WRONLY | O_TRUNC, 0o644); + expect(fs.write(fd, new Uint8Array([0x5a]), null, 1)).toBe(1); + fs.close(fd); + + expect(fs.stat("/lazy").size).toBe(1); + expect(fs.getLazyEntry("/lazy")).toBeNull(); + expect(fs.getLazyEntry("/alias")).toBeNull(); + expect(fs.exportLazyEntries()).toEqual([]); + await expect(fs.ensureMaterialized("/alias")).resolves.toBe(false); + }); + + it("does not apply a delayed fetch to a replacement inode", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + let release!: (value: ArrayBuffer) => void; + const body = new Promise((resolve) => { + release = resolve; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => body, + } as unknown as Response); + + try { + const pending = fs.ensureMaterialized("/lazy"); + await vi.waitFor(() => expect(globalThis.fetch).toHaveBeenCalled()); + + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + peer.unlink("/lazy"); + const replacement = peer.open("/lazy", O_CREAT | O_WRONLY, 0o644); + peer.write(replacement, new Uint8Array([9]), null, 1); + peer.close(replacement); + + release(new Uint8Array([1]).buffer); + await expect(pending).resolves.toBe(false); + + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(9); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("finishes one materialization call after a peer rename during fetch", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + let release!: (value: ArrayBuffer) => void; + const body = new Promise((resolve) => { + release = resolve; + }); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => body, + } as unknown as Response); + + try { + const pending = fs.ensureMaterialized("/lazy"); + await vi.waitFor(() => expect(globalThis.fetch).toHaveBeenCalled()); + peer.rename("/lazy", "/moved"); + release(new Uint8Array([7]).buffer); + + await expect(pending).resolves.toBe(true); + const fd = fs.open("/moved", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + expect(fs.getLazyEntry("/moved")).toBeNull(); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("keeps an empty lazy stub retryable after ENOSPC", async () => { + const originalFetch = globalThis.fetch; + const fs = create(256 * 1024); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 4096); + const filler = fs.open("/filler", O_CREAT | O_WRONLY, 0o644); + const chunk = new Uint8Array(64 * 1024).fill(0xa5); + while (fs.write(filler, chunk, null, chunk.length) > 0) { + // Fill every allocatable block. + } + fs.close(filler); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "4096" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array(4096).fill(7).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/lazy")).rejects.toThrow( + /No space left/, + ); + expect(fs.getLazyEntry("/lazy")).not.toBeNull(); + expect(fs.stat("/lazy").size).toBe(4096); + + fs.unlink("/filler"); + await expect(fs.ensureMaterialized("/lazy")).resolves.toBe(true); + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("does not overwrite a same-inode write that wins after fetch", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([1]).buffer), + } as unknown as Response); + + type ReplaceArgs = [string, number, number, number, Uint8Array]; + const raw = ( + fs as unknown as { + fs: { replaceIfIdentity: (...args: ReplaceArgs) => boolean }; + } + ).fs; + const replace = raw.replaceIfIdentity.bind(raw); + const replaceSpy = vi + .spyOn(raw, "replaceIfIdentity") + .mockImplementation((...args: ReplaceArgs) => { + const fd = peer.open("/lazy", O_WRONLY | O_TRUNC, 0o644); + peer.write(fd, new Uint8Array([9]), null, 1); + peer.close(fd); + return replace(...args); + }); + + try { + await expect(fs.ensureMaterialized("/lazy")).resolves.toBe(false); + const fd = fs.open("/lazy", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(9); + } finally { + replaceSpy.mockRestore(); + globalThis.fetch = originalFetch; + } + }); + + it("materializes through a surviving path renamed by another instance", async () => { + const originalFetch = globalThis.fetch; + const fs = create(); + fs.registerLazyFile("/lazy", "https://example.test/lazy", 1); + const peer = MemoryFileSystem.fromExisting(fs.sharedBuffer); + fs.symlink("/moved", "/indirect"); + peer.rename("/lazy", "/moved"); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers({ "content-length": "1" }), + body: null, + arrayBuffer: () => Promise.resolve(new Uint8Array([7]).buffer), + } as unknown as Response); + + try { + await expect(fs.ensureMaterialized("/indirect")).resolves.toBe(true); + const fd = fs.open("/moved", O_RDONLY, 0); + const byte = new Uint8Array(1); + expect(fs.read(fd, byte, null, 1)).toBe(1); + fs.close(fd); + expect(byte[0]).toBe(7); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); diff --git a/host/test/vfs-image.test.ts b/host/test/vfs-image.test.ts index 44cca1457d..c028d86725 100644 --- a/host/test/vfs-image.test.ts +++ b/host/test/vfs-image.test.ts @@ -18,13 +18,22 @@ function createMemfs(): MemoryFileSystem { return MemoryFileSystem.create(sab); } -function writeFile(mfs: MemoryFileSystem, path: string, data: Uint8Array, mode = 0o644): void { +function writeFile( + mfs: MemoryFileSystem, + path: string, + data: Uint8Array, + mode = 0o644, +): void { // Ensure parent directories exist const parts = path.split("/").filter(Boolean); let current = ""; for (let i = 0; i < parts.length - 1; i++) { current += "/" + parts[i]; - try { mfs.mkdir(current, 0o755); } catch { /* exists */ } + try { + mfs.mkdir(current, 0o755); + } catch { + /* exists */ + } } const fd = mfs.open(path, O_WRONLY | O_CREAT | O_TRUNC, mode); mfs.write(fd, data, null, data.length); @@ -52,6 +61,28 @@ function readDir(mfs: MemoryFileSystem, path: string): string[] { return names.sort(); } +function stripStandaloneLazyIdentity(image: Uint8Array): Uint8Array { + const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const sabLen = view.getUint32(12, true); + const lazyOffset = 16 + sabLen; + const lazyLen = view.getUint32(lazyOffset, true); + const entries = JSON.parse( + new TextDecoder().decode( + image.subarray(lazyOffset + 4, lazyOffset + 4 + lazyLen), + ), + ) as Array<{ generation?: number; dataSequence?: number }>; + for (const entry of entries) { + delete entry.generation; + delete entry.dataSequence; + } + const lazyJson = new TextEncoder().encode(JSON.stringify(entries)); + const legacy = new Uint8Array(lazyOffset + 4 + lazyJson.byteLength); + legacy.set(image.subarray(0, lazyOffset)); + new DataView(legacy.buffer).setUint32(lazyOffset, lazyJson.byteLength, true); + legacy.set(lazyJson, lazyOffset + 4); + return legacy; +} + describe("VFS image save/restore", () => { describe("saveImage + fromImage round-trip", () => { it("preserves a single file", async () => { @@ -86,8 +117,12 @@ describe("VFS image save/restore", () => { const image = await mfs.saveImage(); const restored = MemoryFileSystem.fromImage(image); - expect(readFile(restored, "/usr/local/bin/tool")).toEqual(new Uint8Array([1, 2, 3])); - expect(readFile(restored, "/usr/local/lib/libfoo.so")).toEqual(new Uint8Array([4, 5, 6])); + expect(readFile(restored, "/usr/local/bin/tool")).toEqual( + new Uint8Array([1, 2, 3]), + ); + expect(readFile(restored, "/usr/local/lib/libfoo.so")).toEqual( + new Uint8Array([4, 5, 6]), + ); expect(readFile(restored, "/etc/config")).toEqual(new Uint8Array([7, 8])); // Verify directory listing @@ -149,11 +184,15 @@ describe("VFS image save/restore", () => { // Can write new files writeFile(restored, "/new.txt", new TextEncoder().encode("new")); - expect(new TextDecoder().decode(readFile(restored, "/new.txt"))).toBe("new"); + expect(new TextDecoder().decode(readFile(restored, "/new.txt"))).toBe( + "new", + ); // Can modify existing files writeFile(restored, "/existing.txt", new TextEncoder().encode("updated")); - expect(new TextDecoder().decode(readFile(restored, "/existing.txt"))).toBe("updated"); + expect( + new TextDecoder().decode(readFile(restored, "/existing.txt")), + ).toBe("updated"); }); }); @@ -161,13 +200,20 @@ describe("VFS image save/restore", () => { it("preserves lazy file metadata by default", async () => { const mfs = createMemfs(); writeFile(mfs, "/real.txt", new TextEncoder().encode("real content")); - mfs.registerLazyFile("/bin/lazy-tool", "http://example.com/tool.wasm", 12345, 0o755); + mfs.registerLazyFile( + "/bin/lazy-tool", + "http://example.com/tool.wasm", + 12345, + 0o755, + ); const image = await mfs.saveImage(); const restored = MemoryFileSystem.fromImage(image); // Real file content preserved - expect(new TextDecoder().decode(readFile(restored, "/real.txt"))).toBe("real content"); + expect(new TextDecoder().decode(readFile(restored, "/real.txt"))).toBe( + "real content", + ); // Lazy file metadata preserved — stat reports declared size const st = restored.stat("/bin/lazy-tool"); @@ -182,6 +228,27 @@ describe("VFS image save/restore", () => { expect(entries[0].size).toBe(12345); }); + it("restores identity-less lazy metadata only from its coherent legacy image", async () => { + const mfs = createMemfs(); + mfs.registerLazyFile( + "/bin/legacy-tool", + "http://example.com/legacy.wasm", + 54321, + 0o755, + ); + + const restored = MemoryFileSystem.fromImage( + stripStandaloneLazyIdentity(await mfs.saveImage()), + ); + + expect(restored.getLazyEntry("/bin/legacy-tool")).toMatchObject({ + path: "/bin/legacy-tool", + url: "http://example.com/legacy.wasm", + size: 54321, + }); + expect(restored.stat("/bin/legacy-tool").size).toBe(54321); + }); + it("preserves multiple lazy files", async () => { const mfs = createMemfs(); mfs.registerLazyFile("/bin/a", "http://example.com/a.wasm", 100); @@ -200,29 +267,43 @@ describe("VFS image save/restore", () => { ]); }); - it("materializeAll clears lazy entries from image", async () => { + it("reconciles a peer-renamed lazy path before saving", async () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const fsA = MemoryFileSystem.create(sab); + const fsB = MemoryFileSystem.fromExisting(sab); + const url = "http://example.com/lazy.wasm"; + + fsA.registerLazyFile("/lazy", url, 12_345, 0o755); + fsB.rename("/lazy", "/moved"); + + const restored = MemoryFileSystem.fromImage(await fsA.saveImage()); + expect(() => restored.stat("/lazy")).toThrow(/No such file/); + expect(restored.stat("/moved").size).toBe(12_345); + expect(restored.getLazyEntry("/moved")).toMatchObject({ + path: "/moved", + url, + size: 12_345, + }); + expect(restored.exportLazyEntries()).toHaveLength(1); + }); + + it("explicit content replacement clears lazy entries from the image", async () => { const mfs = createMemfs(); - // Register a lazy file and manually write content to simulate materialization - // (can't actually fetch in tests, so we simulate by writing content before save) - mfs.registerLazyFile("/bin/tool", "http://example.com/tool.wasm", 5000, 0o755); + mfs.registerLazyFile( + "/bin/tool", + "http://example.com/tool.wasm", + 5000, + 0o755, + ); - // Manually write content to the file (simulating what ensureMaterialized does) const toolContent = new Uint8Array(100); for (let i = 0; i < 100; i++) toolContent[i] = i; const fd = mfs.open("/bin/tool", O_WRONLY | O_CREAT | O_TRUNC, 0o755); mfs.write(fd, toolContent, null, toolContent.length); mfs.close(fd); - // Clear the lazy entry manually to simulate materialization - // (ensureMaterialized would do this via fetch + delete) - const entries = mfs.exportLazyEntries(); - expect(entries).toHaveLength(1); - - // Instead, test that a save without lazy entries works correctly - // by creating a fresh mfs with no lazy files - const mfs2 = createMemfs(); - writeFile(mfs2, "/bin/tool", toolContent, 0o755); - const image = await mfs2.saveImage(); + expect(mfs.exportLazyEntries()).toHaveLength(0); + const image = await mfs.saveImage(); const restored = MemoryFileSystem.fromImage(image); expect(restored.exportLazyEntries()).toHaveLength(0); @@ -234,7 +315,11 @@ describe("VFS image save/restore", () => { writeFile(mfs, "/test.txt", new TextEncoder().encode("test")); const image = await mfs.saveImage(); - const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const view = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); const flags = view.getUint32(8, true); expect(flags & 1).toBe(0); // no lazy flag }); @@ -244,7 +329,11 @@ describe("VFS image save/restore", () => { mfs.registerLazyFile("/bin/tool", "http://example.com/tool.wasm", 1000); const image = await mfs.saveImage(); - const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const view = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); const flags = view.getUint32(8, true); expect(flags & 1).toBe(1); // lazy flag set }); @@ -261,7 +350,11 @@ describe("VFS image save/restore", () => { }); const image = await mfs.saveImage(); - const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const view = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); const flags = view.getUint32(8, true); expect(flags & 4).toBe(4); // metadata flag set @@ -277,8 +370,9 @@ describe("VFS image save/restore", () => { kernelAbi: 11, createdBy: "vfs-image.test", }); - expect(new TextDecoder().decode(readFile(restored, "/bin/tool"))) - .toBe("\0asm"); + expect(new TextDecoder().decode(readFile(restored, "/bin/tool"))).toBe( + "\0asm", + ); }); it("preserves metadata when a restored image is saved again", async () => { @@ -304,7 +398,11 @@ describe("VFS image save/restore", () => { const restored = MemoryFileSystem.fromImage(image); const cleared = await restored.saveImage({ metadata: null }); - const view = new DataView(cleared.buffer, cleared.byteOffset, cleared.byteLength); + const view = new DataView( + cleared.buffer, + cleared.byteOffset, + cleared.byteLength, + ); const flags = view.getUint32(8, true); expect(flags & 4).toBe(0); expect(MemoryFileSystem.readImageMetadata(cleared)).toBeNull(); @@ -322,10 +420,12 @@ describe("VFS image save/restore", () => { version: 1, kernelAbi: ABI_VERSION, }); - expect(MemoryFileSystem.fromImage(compressed).getImageMetadata()).toEqual({ - version: 1, - kernelAbi: ABI_VERSION, - }); + expect(MemoryFileSystem.fromImage(compressed).getImageMetadata()).toEqual( + { + version: 1, + kernelAbi: ABI_VERSION, + }, + ); }); it("rejects malformed metadata declarations", async () => { @@ -345,11 +445,16 @@ describe("VFS image save/restore", () => { metadata: { version: 1, kernelAbi: olderAbi }, }); - expect(() => MemoryFileSystem.assertImageKernelAbi(image, olderAbi)).not.toThrow(); - expect(() => MemoryFileSystem.assertImageKernelAbi(image, ABI_VERSION, "test image")) - .toThrow(new RegExp( + expect(() => + MemoryFileSystem.assertImageKernelAbi(image, olderAbi), + ).not.toThrow(); + expect(() => + MemoryFileSystem.assertImageKernelAbi(image, ABI_VERSION, "test image"), + ).toThrow( + new RegExp( `test image requires kernel ABI ${olderAbi}.*running kernel is ABI ${ABI_VERSION}`, - )); + ), + ); }); }); @@ -369,20 +474,28 @@ describe("VFS image save/restore", () => { expect(compressed[3]).toBe(0xfd); const restored = MemoryFileSystem.fromImage(compressed); - expect(new TextDecoder().decode(readFile(restored, "/hello.txt"))).toBe("compressed!"); - expect(readFile(restored, "/etc/config")).toEqual(new Uint8Array([1, 2, 3, 4])); + expect(new TextDecoder().decode(readFile(restored, "/hello.txt"))).toBe( + "compressed!", + ); + expect(readFile(restored, "/etc/config")).toEqual( + new Uint8Array([1, 2, 3, 4]), + ); }); it("fromImage with maxByteLength still works on zstd-wrapped image", async () => { const mfs = createMemfs(); writeFile(mfs, "/data.txt", new TextEncoder().encode("hi")); - const compressed = new Uint8Array(zstdCompressSync(await mfs.saveImage())); + const compressed = new Uint8Array( + zstdCompressSync(await mfs.saveImage()), + ); const restored = MemoryFileSystem.fromImage(compressed, { maxByteLength: 16 * 1024 * 1024, }); expect(restored.sharedBuffer.maxByteLength).toBe(16 * 1024 * 1024); - expect(new TextDecoder().decode(readFile(restored, "/data.txt"))).toBe("hi"); + expect(new TextDecoder().decode(readFile(restored, "/data.txt"))).toBe( + "hi", + ); }); it("saveImage writes a .vfs.zst that fromImage can restore", async () => { @@ -411,8 +524,9 @@ describe("VFS image save/restore", () => { const restored = MemoryFileSystem.fromImage(onDisk, { maxByteLength: 8 * 1024 * 1024, }); - expect(new TextDecoder().decode(readFile(restored, "/etc/hostname"))) - .toBe("wasmbox\n"); + expect( + new TextDecoder().decode(readFile(restored, "/etc/hostname")), + ).toBe("wasmbox\n"); const restoredBig = readFile(restored, "/usr/lib/libfoo.so"); expect(restoredBig.length).toBe(big.length); expect(restoredBig).toEqual(big); @@ -438,7 +552,9 @@ describe("VFS image save/restore", () => { it("rejects image with bad magic", () => { const bad = new Uint8Array(32); new DataView(bad.buffer).setUint32(0, 0xdeadbeef, true); - expect(() => MemoryFileSystem.fromImage(bad)).toThrow("Bad VFS image magic"); + expect(() => MemoryFileSystem.fromImage(bad)).toThrow( + "Bad VFS image magic", + ); }); it("rejects image with wrong version", () => { @@ -446,7 +562,9 @@ describe("VFS image save/restore", () => { const view = new DataView(bad.buffer); view.setUint32(0, 0x56465349, true); // VFSI magic view.setUint32(4, 99, true); // bad version - expect(() => MemoryFileSystem.fromImage(bad)).toThrow("Unsupported VFS image version"); + expect(() => MemoryFileSystem.fromImage(bad)).toThrow( + "Unsupported VFS image version", + ); }); it("rejects truncated image", () => { @@ -460,13 +578,19 @@ describe("VFS image save/restore", () => { }); it("rejects image that is too small", () => { - expect(() => MemoryFileSystem.fromImage(new Uint8Array(4))).toThrow("too small"); + expect(() => MemoryFileSystem.fromImage(new Uint8Array(4))).toThrow( + "too small", + ); }); it("image has correct magic and version", async () => { const mfs = createMemfs(); const image = await mfs.saveImage(); - const view = new DataView(image.buffer, image.byteOffset, image.byteLength); + const view = new DataView( + image.buffer, + image.byteOffset, + image.byteLength, + ); expect(view.getUint32(0, true)).toBe(0x56465349); // "VFSI" expect(view.getUint32(4, true)).toBe(1); // version 1 }); @@ -504,7 +628,9 @@ describe("VFS image save/restore", () => { const image = await mfs.saveImage(); const maxBytes = 16 * 1024 * 1024; - const restored = MemoryFileSystem.fromImage(image, { maxByteLength: maxBytes }); + const restored = MemoryFileSystem.fromImage(image, { + maxByteLength: maxBytes, + }); const buf = restored.sharedBuffer; expect(buf).toBeInstanceOf(SharedArrayBuffer); // The SAB should have maxByteLength set (growable) @@ -558,12 +684,18 @@ describe("VFS image save/restore", () => { writeFile(mfs, "/original.txt", new TextEncoder().encode("data")); const image = await mfs.saveImage(); - const restored = MemoryFileSystem.fromImage(image, { maxByteLength: 32 * 1024 * 1024 }); + const restored = MemoryFileSystem.fromImage(image, { + maxByteLength: 32 * 1024 * 1024, + }); // Can read existing files - expect(new TextDecoder().decode(readFile(restored, "/original.txt"))).toBe("data"); + expect( + new TextDecoder().decode(readFile(restored, "/original.txt")), + ).toBe("data"); // Can write new files writeFile(restored, "/new.txt", new TextEncoder().encode("new data")); - expect(new TextDecoder().decode(readFile(restored, "/new.txt"))).toBe("new data"); + expect(new TextDecoder().decode(readFile(restored, "/new.txt"))).toBe( + "new data", + ); }); it("without maxByteLength creates a non-growable SAB", async () => { @@ -588,7 +720,9 @@ describe("VFS image save/restore", () => { expect(rebased.sharedBuffer.byteLength).toBeGreaterThan(16 * 1024 * 1024); expect(rebased.sharedBuffer.maxByteLength).toBe(maxBytes); expect(stats.blocks * stats.bsize).toBe(maxBytes); - expect(new TextDecoder().decode(readFile(rebased, "/data.txt"))).toBe("base"); + expect(new TextDecoder().decode(readFile(rebased, "/data.txt"))).toBe( + "base", + ); }); it("raises the filesystem max beyond the source image superblock cap", async () => { @@ -605,30 +739,43 @@ describe("VFS image save/restore", () => { const restored = MemoryFileSystem.fromImage(image, { maxByteLength: rebaseMaxBytes, }); - expect(restored.statfs("/").blocks * restored.statfs("/").bsize).toBe(imageMaxBytes); + expect(restored.statfs("/").blocks * restored.statfs("/").bsize).toBe( + imageMaxBytes, + ); const rebased = restored.rebaseToNewFileSystem(rebaseMaxBytes); const stats = rebased.statfs("/"); expect(rebased.sharedBuffer.maxByteLength).toBe(rebaseMaxBytes); expect(stats.blocks * stats.bsize).toBe(rebaseMaxBytes); - expect(new TextDecoder().decode(readFile(rebased, "/data.txt"))).toBe("base"); + expect(new TextDecoder().decode(readFile(rebased, "/data.txt"))).toBe( + "base", + ); const rebasedImage = await rebased.saveImage(); const rerestored = MemoryFileSystem.fromImage(rebasedImage, { maxByteLength: rebaseMaxBytes, }); const rerestoredStats = rerestored.statfs("/"); - expect(rerestoredStats.blocks * rerestoredStats.bsize).toBe(rebaseMaxBytes); + expect(rerestoredStats.blocks * rerestoredStats.bsize).toBe( + rebaseMaxBytes, + ); }); it("preserves lazy file metadata without materializing stubs", async () => { const mfs = createMemfs(); writeFile(mfs, "/real.txt", new TextEncoder().encode("real")); - mfs.registerLazyFile("/bin/lazy-tool", "http://example.com/tool.wasm", 5_000_000, 0o755); + mfs.registerLazyFile( + "/bin/lazy-tool", + "http://example.com/tool.wasm", + 5_000_000, + 0o755, + ); const rebased = mfs.rebaseToNewFileSystem(16 * 1024 * 1024); - expect(new TextDecoder().decode(readFile(rebased, "/real.txt"))).toBe("real"); + expect(new TextDecoder().decode(readFile(rebased, "/real.txt"))).toBe( + "real", + ); expect(rebased.stat("/bin/lazy-tool").size).toBe(5_000_000); expect(rebased.stat("/bin/lazy-tool").mode & 0o777).toBe(0o755); expect(rebased.exportLazyEntries()).toMatchObject([ @@ -658,7 +805,9 @@ describe("VFS image save/restore", () => { writeFile(mfs, "/test.txt", new TextEncoder().encode("modified")); // Restored should still have original content - expect(new TextDecoder().decode(readFile(restored, "/test.txt"))).toBe("original"); + expect(new TextDecoder().decode(readFile(restored, "/test.txt"))).toBe( + "original", + ); }); it("modifications to restored filesystem don't affect original", async () => { @@ -673,7 +822,9 @@ describe("VFS image save/restore", () => { writeFile(restored, "/new.txt", new TextEncoder().encode("new")); // Original should be untouched - expect(new TextDecoder().decode(readFile(mfs, "/test.txt"))).toBe("original"); + expect(new TextDecoder().decode(readFile(mfs, "/test.txt"))).toBe( + "original", + ); expect(() => mfs.stat("/new.txt")).toThrow(); }); @@ -690,7 +841,9 @@ describe("VFS image save/restore", () => { expect(new TextDecoder().decode(readFile(r1, "/data.txt"))).toBe("r1"); expect(new TextDecoder().decode(readFile(r2, "/data.txt"))).toBe("r2"); - expect(new TextDecoder().decode(readFile(mfs, "/data.txt"))).toBe("shared"); + expect(new TextDecoder().decode(readFile(mfs, "/data.txt"))).toBe( + "shared", + ); }); }); }); diff --git a/host/test/vfs.test.ts b/host/test/vfs.test.ts index ade23b0429..aede93eb94 100644 --- a/host/test/vfs.test.ts +++ b/host/test/vfs.test.ts @@ -1,5 +1,11 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { mkdtempSync, writeFileSync, readFileSync, rmSync } from "node:fs"; +import { + mkdirSync, + mkdtempSync, + writeFileSync, + readFileSync, + rmSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { VirtualPlatformIO } from "../src/vfs/vfs"; @@ -397,13 +403,24 @@ describe("VirtualPlatformIO cross-mount rename (EXDEV)", () => { describe("HostFileSystem path traversal", () => { it("rejects paths that escape rootPath", () => { - const hfs = new HostFileSystem("/tmp/sandbox"); - expect(() => hfs.stat("/../../../etc/passwd")).toThrow("EACCES"); + const root = mkdtempSync(join(tmpdir(), "kandelo-host-fs-traversal-")); + try { + const hfs = new HostFileSystem(root); + expect(() => hfs.stat("/../../../etc/passwd")).toThrow("EACCES"); + } finally { + rmSync(root, { recursive: true, force: true }); + } }); it("rejects paths with embedded .. sequences", () => { - const hfs = new HostFileSystem("/tmp/sandbox"); - expect(() => hfs.stat("/subdir/../../etc/passwd")).toThrow("EACCES"); + const root = mkdtempSync(join(tmpdir(), "kandelo-host-fs-traversal-")); + try { + mkdirSync(join(root, "subdir")); + const hfs = new HostFileSystem(root); + expect(() => hfs.stat("/subdir/../../etc/passwd")).toThrow("EACCES"); + } finally { + rmSync(root, { recursive: true, force: true }); + } }); }); @@ -545,6 +562,32 @@ describe("MemoryFileSystem", () => { mfs.close(fd); }); + it("keeps large-directory indexes coherent across SharedFS instances", () => { + const sab = new SharedArrayBuffer(8 * 1024 * 1024); + const first = MemoryFileSystem.create(sab); + const second = MemoryFileSystem.fromExisting(sab); + first.mkdir("/bulk", 0o755); + + const names: string[] = []; + for (let i = 0; i < 340; i++) { + const name = `/bulk/${String(i).padStart(4, "0")}-${"x".repeat(180)}`; + names.push(name); + const fd = first.open(name, O_CREAT | O_RDWR, 0o644); + first.close(fd); + } + + // Populate the first mount's index, then reuse a deleted slot through a + // second mount without changing the directory's byte size. + expect(first.stat(names.at(-1)!).mode & 0xf000).toBe(0x8000); + second.unlink(names[100]); + const replacement = `/bulk/repl-${"y".repeat(180)}`; + const replacementFd = second.open(replacement, O_CREAT | O_RDWR, 0o644); + second.close(replacementFd); + + expect(first.stat(replacement).mode & 0xf000).toBe(0x8000); + expect(() => first.stat(names[100])).toThrow(/No such file/); + }); + it("honors O_CREAT|O_EXCL by failing when the final path already exists", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); @@ -701,6 +744,31 @@ describe("MemoryFileSystem", () => { expect(() => mfs.stat("/new-parent/child")).toThrow(/No such file/); }); + it("rejects rename and rmdir operands ending in dot or dot-dot", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + mfs.mkdir("/dir", 0o755); + mfs.mkdir("/dir/child", 0o755); + + expect(() => mfs.rename("/dir/.", "/moved")).toThrow(/Invalid argument/); + expect(() => mfs.rename("/dir/child", "/dir/..")).toThrow(/Invalid argument/); + expect(() => mfs.rmdir("/dir/.")).toThrow(/Invalid argument/); + expect(() => mfs.rmdir("/dir/child/..")).toThrow(/Invalid argument/); + expect(mfs.stat("/dir/child").mode & 0xf000).toBe(0x4000); + }); + + it("chmod and fchmod preserve the inode file type", () => { + const sab = new SharedArrayBuffer(4 * 1024 * 1024); + const mfs = MemoryFileSystem.create(sab); + const fd = mfs.open("/regular", O_CREAT | O_RDWR, 0o644); + + mfs.chmod("/regular", 0o040755); + expect(mfs.stat("/regular").mode & 0xf000).toBe(0x8000); + mfs.fchmod(fd, 0o040700); + expect(mfs.fstat(fd).mode & 0xf000).toBe(0x8000); + mfs.close(fd); + }); + it("keeps an unlinked open file alive until close", () => { const sab = new SharedArrayBuffer(4 * 1024 * 1024); const mfs = MemoryFileSystem.create(sab); diff --git a/packages/registry/erlang-vfs/build.toml b/packages/registry/erlang-vfs/build.toml index 071017f82f..2971c033f9 100644 --- a/packages/registry/erlang-vfs/build.toml +++ b/packages/registry/erlang-vfs/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/erlang-vfs/build-erlang-vfs.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/kandelo-sdk/build.toml b/packages/registry/kandelo-sdk/build.toml index a993a28bcd..a34738eeaa 100644 --- a/packages/registry/kandelo-sdk/build.toml +++ b/packages/registry/kandelo-sdk/build.toml @@ -26,7 +26,7 @@ inputs = [ ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/mariadb-test/build.toml b/packages/registry/mariadb-test/build.toml index 8a6904e518..6a762ae933 100644 --- a/packages/registry/mariadb-test/build.toml +++ b/packages/registry/mariadb-test/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/mariadb-test/build-mariadb-test.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 1 +revision = 2 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/mariadb-vfs/build.toml b/packages/registry/mariadb-vfs/build.toml index 5a456a5ee6..7e7be2a374 100644 --- a/packages/registry/mariadb-vfs/build.toml +++ b/packages/registry/mariadb-vfs/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/mariadb-vfs/build-mariadb-vfs.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/node-vfs/build.toml b/packages/registry/node-vfs/build.toml index b7d7d96551..8df934459d 100644 --- a/packages/registry/node-vfs/build.toml +++ b/packages/registry/node-vfs/build.toml @@ -21,7 +21,7 @@ inputs = [ ] repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 7 +revision = 8 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/perl-vfs/build.toml b/packages/registry/perl-vfs/build.toml index ea96416dc6..539cae8c7a 100644 --- a/packages/registry/perl-vfs/build.toml +++ b/packages/registry/perl-vfs/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/perl-vfs/build-perl-vfs.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/python-vfs/build.toml b/packages/registry/python-vfs/build.toml index 41a8c35234..aa2d2379af 100644 --- a/packages/registry/python-vfs/build.toml +++ b/packages/registry/python-vfs/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/python-vfs/build-python-vfs.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 2 +revision = 3 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" From 348da0d4667e29f7e0894ed69870898ef6b4bcf9 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 07:49:08 -0400 Subject: [PATCH 25/29] fix: make rootfs services authoritative in derived PHP images Use the canonical services database and dinit image helpers when deriving browser VFS artifacts, align WordPress and LAMP on PHP 8.3.15, and invalidate every serialized image whose inputs changed. --- MANIFEST | 5 ++- examples/getpwent_smoke.c | 20 ++++++++++++ host/test/dinit-image-helpers.test.ts | 39 +++++++++++++++++++++++ host/test/getpwent.test.ts | 9 +++++- images/rootfs/etc/services | 4 ++- images/vfs/scripts/dinit-image-helpers.ts | 29 ++++++++++------- images/vfs/scripts/shell-vfs-build.ts | 31 ------------------ packages/registry/lamp/build.toml | 3 +- packages/registry/lamp/package.toml | 2 +- packages/registry/rootfs/build.toml | 2 +- packages/registry/shell/build.toml | 2 +- packages/registry/wordpress/build.toml | 3 +- packages/registry/wordpress/package.toml | 2 +- 13 files changed, 98 insertions(+), 53 deletions(-) create mode 100644 host/test/dinit-image-helpers.test.ts diff --git a/MANIFEST b/MANIFEST index 27d0251ab6..0ee89fe22e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -51,9 +51,8 @@ /etc/os-release f 0644 0 0 /etc/profile f 0644 0 0 /etc/motd f 0644 0 0 -# services matches the ad-hoc table in -# apps/browser-demos/lib/browser-kernel.ts byte-for-byte so the cutover -# is invisible to getservbyname/getservbyport callers. +# The rootfs file is the authoritative services database. Rootfs-derived +# images inherit it rather than rebuilding or synthesizing a second table. /etc/services f 0644 0 0 # ── Shells ───────────────────────────────────────────────────────── diff --git a/examples/getpwent_smoke.c b/examples/getpwent_smoke.c index 9e32c948d8..6d01d4ea56 100644 --- a/examples/getpwent_smoke.c +++ b/examples/getpwent_smoke.c @@ -83,6 +83,21 @@ static int check_servbyname(const char *name, const char *proto, int expect_port return port == (unsigned)expect_port ? 0 : 1; } +static int check_service_query(const char *query, const char *proto, + const char *expect_name, int expect_port) { + struct servent *se = getservbyname(query, proto); + if (se == NULL) { + printf("SERVENT query=%s proto=%s result=NULL\n", query, proto); + return 1; + } + unsigned short port = ((unsigned short)se->s_port << 8) | + ((unsigned short)se->s_port >> 8); + printf("SERVENT query=%s name=%s proto=%s port=%u\n", + query, se->s_name, proto, (unsigned)port); + return (strcmp(se->s_name, expect_name) == 0 && + port == (unsigned)expect_port) ? 0 : 1; +} + int main(void) { int rc = 0; rc |= check_pwent_iteration(); @@ -94,6 +109,11 @@ int main(void) { rc |= check_grent_iteration(); rc |= check_servbyname("ssh", "tcp", 22); rc |= check_servbyname("http", "tcp", 80); + rc |= check_service_query("www", "tcp", "www", 80); + rc |= check_service_query("www-http", "tcp", "www-http", 80); + rc |= check_service_query("https", "tcp", "https", 443); + rc |= check_service_query("mysql", "tcp", "mysql", 3306); + rc |= check_service_query("postgresql", "tcp", "postgresql", 5432); printf("DONE rc=%d\n", rc); return rc; } diff --git a/host/test/dinit-image-helpers.test.ts b/host/test/dinit-image-helpers.test.ts new file mode 100644 index 0000000000..4f92296d2c --- /dev/null +++ b/host/test/dinit-image-helpers.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { findRepoRoot } from "../src/binary-resolver"; +import { MemoryFileSystem } from "../src/vfs/memory-fs"; +import { addDinitBaseSystemFiles } from "../../images/vfs/scripts/dinit-image-helpers"; + +const O_RDONLY = 0; + +function readGuestFile(fs: MemoryFileSystem, path: string): string { + const size = fs.stat(path).size; + const fd = fs.open(path, O_RDONLY, 0); + try { + const bytes = new Uint8Array(size); + const count = fs.read(fd, bytes, null, bytes.byteLength); + return new TextDecoder().decode(bytes.subarray(0, count)); + } finally { + fs.close(fd); + } +} + +describe("dinit-derived image system databases", () => { + it("copies the authoritative rootfs services database without reducing aliases", () => { + const fs = MemoryFileSystem.create( + new SharedArrayBuffer(4 * 1024 * 1024), + ); + addDinitBaseSystemFiles(fs); + + const source = readFileSync( + join(findRepoRoot(), "images", "rootfs", "etc", "services"), + "utf8", + ); + const derived = readGuestFile(fs, "/etc/services"); + + expect(derived).toBe(source); + expect(derived).toContain("www www-http"); + expect(derived).toContain("postgresql\t5432/tcp"); + }); +}); diff --git a/host/test/getpwent.test.ts b/host/test/getpwent.test.ts index e7508b06ea..1eb38a7eb5 100644 --- a/host/test/getpwent.test.ts +++ b/host/test/getpwent.test.ts @@ -91,7 +91,7 @@ describe.skipIf(!haveSmoke || !haveRootfs)("getpwent via rootfs.vfs mount", () = expect(result.stdout).toContain("GRENT count=8"); }); - it("resolves service names from /etc/services via getservbyname", async () => { + it("resolves canonical service names and aliases from the rootfs image", async () => { const result = await runCentralizedProgram({ programPath: smokeWasm, argv: ["getpwent_smoke"], @@ -101,5 +101,12 @@ describe.skipIf(!haveSmoke || !haveRootfs)("getpwent via rootfs.vfs mount", () = expect(result.exitCode, result.stderr || result.stdout).toBe(0); expect(result.stdout).toContain("SERV name=ssh proto=tcp port=22"); expect(result.stdout).toContain("SERV name=http proto=tcp port=80"); + expect(result.stdout).toContain("SERVENT query=www name=www proto=tcp port=80"); + expect(result.stdout).toContain("SERVENT query=www-http name=www-http proto=tcp port=80"); + expect(result.stdout).toContain("SERVENT query=https name=https proto=tcp port=443"); + expect(result.stdout).toContain("SERVENT query=mysql name=mysql proto=tcp port=3306"); + expect(result.stdout).toContain( + "SERVENT query=postgresql name=postgresql proto=tcp port=5432", + ); }); }); diff --git a/images/rootfs/etc/services b/images/rootfs/etc/services index 8ba68d7efc..bd91937a30 100644 --- a/images/rootfs/etc/services +++ b/images/rootfs/etc/services @@ -13,7 +13,7 @@ domain 53/tcp domain 53/udp gopher 70/tcp finger 79/tcp -http 80/tcp www +http 80/tcp www www-http pop3 110/tcp pop-3 nntp 119/tcp readnews untp ntp 123/udp @@ -22,3 +22,5 @@ snmp 161/udp https 443/tcp imaps 993/tcp pop3s 995/tcp +mysql 3306/tcp +postgresql 5432/tcp diff --git a/images/vfs/scripts/dinit-image-helpers.ts b/images/vfs/scripts/dinit-image-helpers.ts index d220ce0339..ae10aedd85 100644 --- a/images/vfs/scripts/dinit-image-helpers.ts +++ b/images/vfs/scripts/dinit-image-helpers.ts @@ -146,12 +146,23 @@ const ETC_HOSTS = [ "", ].join("\n"); -const ETC_SERVICES = [ - "http\t\t80/tcp\t\twww", - "https\t\t443/tcp", - "mysql\t\t3306/tcp", - "", -].join("\n"); +const ETC_SERVICES = readFileSync( + join(REPO_ROOT, "images", "rootfs", "etc", "services"), + "utf8", +); + +/** + * Install the account and network databases shared by dinit-based images. + * `/etc/services` comes from the rootfs source so derived images cannot drift + * into a second, smaller service-name contract. + */ +export function addDinitBaseSystemFiles(fs: MemoryFileSystem): void { + ensureDirRecursive(fs, "/etc"); + writeVfsFile(fs, "/etc/passwd", ETC_PASSWD); + writeVfsFile(fs, "/etc/group", ETC_GROUP); + writeVfsFile(fs, "/etc/hosts", ETC_HOSTS); + writeVfsFile(fs, "/etc/services", ETC_SERVICES); +} /** * Options for {@link addDinitInit}. The defaults set up an implicit @@ -264,11 +275,7 @@ export function addDinitInit( // Basic rootfs files. Most Unix daemons expect these to exist at // startup; missing them is the usual cause of "started but exits 1 // silently" failures. - ensureDirRecursive(fs, "/etc"); - writeVfsFile(fs, "/etc/passwd", ETC_PASSWD); - writeVfsFile(fs, "/etc/group", ETC_GROUP); - writeVfsFile(fs, "/etc/hosts", ETC_HOSTS); - writeVfsFile(fs, "/etc/services", ETC_SERVICES); + addDinitBaseSystemFiles(fs); // Standard runtime/log dirs ensureDirRecursive(fs, "/var/log"); diff --git a/images/vfs/scripts/shell-vfs-build.ts b/images/vfs/scripts/shell-vfs-build.ts index 5136051214..fb1087c427 100644 --- a/images/vfs/scripts/shell-vfs-build.ts +++ b/images/vfs/scripts/shell-vfs-build.ts @@ -162,37 +162,6 @@ function populateSystem(fs: MemoryFileSystem): void { fs.chown("/home/user", 1000, 1000); populateNetHackPlayground(fs); - // /etc/services — required for getservbyname/getservbyport calls in - // nginx/php-fpm/MariaDB. Harmless in Shell-only builds. - const services = [ - "tcpmux\t\t1/tcp", - "echo\t\t7/tcp", - "echo\t\t7/udp", - "discard\t\t9/tcp\t\tsink null", - "discard\t\t9/udp\t\tsink null", - "ftp-data\t20/tcp", - "ftp\t\t21/tcp", - "ssh\t\t22/tcp", - "telnet\t\t23/tcp", - "smtp\t\t25/tcp\t\tmail", - "nicname\t\t43/tcp\t\twhois", - "domain\t\t53/tcp", - "domain\t\t53/udp", - "gopher\t\t70/tcp", - "finger\t\t79/tcp", - "http\t\t80/tcp\t\twww", - "pop3\t\t110/tcp\t\tpop-3", - "nntp\t\t119/tcp\t\treadnews untp", - "ntp\t\t123/udp", - "imap\t\t143/tcp\t\timap2", - "snmp\t\t161/udp", - "https\t\t443/tcp", - "imaps\t\t993/tcp", - "pop3s\t\t995/tcp", - "mysql\t\t3306/tcp", - ].join("\n") + "\n"; - writeVfsFile(fs, "/etc/services", services); - const gitconfig = [ "[maintenance]", "\tauto = false", diff --git a/packages/registry/lamp/build.toml b/packages/registry/lamp/build.toml index cfa8902050..3153c80868 100644 --- a/packages/registry/lamp/build.toml +++ b/packages/registry/lamp/build.toml @@ -8,6 +8,7 @@ inputs = [ "images/vfs/scripts/build-lamp-vfs-image.sh", "images/vfs/scripts/build-lamp-vfs-image.ts", "images/vfs/scripts/dinit-image-helpers.ts", + "images/rootfs/etc/services", "images/vfs/scripts/kandelo-demo-config.ts", "images/vfs/scripts/kandelo-demo-guides.ts", "images/vfs/scripts/opcache-prewarm.ts", @@ -31,7 +32,7 @@ inputs = [ ] repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 7 +revision = 8 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/lamp/package.toml b/packages/registry/lamp/package.toml index d898b8a55c..b7ad6ee9ce 100644 --- a/packages/registry/lamp/package.toml +++ b/packages/registry/lamp/package.toml @@ -14,7 +14,7 @@ depends_on = [ "shell@0.1.0", "mariadb@10.5.28", "nginx@1.24.0", - "php@8.3.2", + "php@8.3.15", "dinit@0.19.4", "msmtpd@1.8.32", ] diff --git a/packages/registry/rootfs/build.toml b/packages/registry/rootfs/build.toml index 2d5643dbc7..13ab3e23dc 100644 --- a/packages/registry/rootfs/build.toml +++ b/packages/registry/rootfs/build.toml @@ -19,7 +19,7 @@ inputs = [ ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 5 +revision = 6 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/shell/build.toml b/packages/registry/shell/build.toml index 833d68a5f8..f0261c1625 100644 --- a/packages/registry/shell/build.toml +++ b/packages/registry/shell/build.toml @@ -24,7 +24,7 @@ inputs = [ ] repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 10 +revision = 11 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/wordpress/build.toml b/packages/registry/wordpress/build.toml index 4b8c21504d..e37037b9c4 100644 --- a/packages/registry/wordpress/build.toml +++ b/packages/registry/wordpress/build.toml @@ -6,6 +6,7 @@ inputs = [ "images/vfs/scripts/build-wp-vfs-image.sh", "images/vfs/scripts/build-wp-vfs-image.ts", "images/vfs/scripts/dinit-image-helpers.ts", + "images/rootfs/etc/services", "images/vfs/scripts/kandelo-demo-config.ts", "images/vfs/scripts/kandelo-demo-guides.ts", "images/vfs/scripts/opcache-prewarm.ts", @@ -28,7 +29,7 @@ inputs = [ ] repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 7 +revision = 8 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/wordpress/package.toml b/packages/registry/wordpress/package.toml index d872cb3a4f..6281590bc5 100644 --- a/packages/registry/wordpress/package.toml +++ b/packages/registry/wordpress/package.toml @@ -9,7 +9,7 @@ kernel_abi = 7 depends_on = [ "shell@0.1.0", "nginx@1.24.0", - "php@8.3.2", + "php@8.3.15", "dinit@0.19.4", "msmtpd@1.8.32", ] From 583bcec0952ca2a3a511ff7ff033cf92f02300df Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 07:49:23 -0400 Subject: [PATCH 26/29] fix: give PHP shutdown a fresh ordinary timeout phase Keep the soft-plus-hard deadline for unresponsive Wasm execution, but reset shutdown callbacks to a new soft timeout so ordinary PHP cleanup remains bounded without being killed immediately. --- packages/registry/php/build-php.sh | 16 +++++----------- packages/registry/php/build.toml | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/registry/php/build-php.sh b/packages/registry/php/build-php.sh index 144b5ce14b..1e30253cee 100755 --- a/packages/registry/php/build-php.sh +++ b/packages/registry/php/build-php.sh @@ -373,14 +373,6 @@ replace_once( "\t}\n" "}\n" "\n" - "static zend_always_inline void zend_wasm_arm_hard_timeout(zend_long seconds)\n" - "{\n" - "\tzend_hrtime_t now = zend_hrtime();\n" - "\tzend_hrtime_t hard_ns = zend_wasm_timeout_seconds_to_ns(seconds);\n" - "\tzend_wasm_timeout_deadline = 0;\n" - "\tzend_wasm_hard_timeout_deadline = zend_wasm_deadline_after(now, hard_ns);\n" - "}\n" - "\n" "static zend_always_inline bool zend_wasm_hard_timeout_expired(void)\n" "{\n" "\tzend_hrtime_t now;\n" @@ -488,11 +480,13 @@ replace_once( "\t}\n" "# endif\n" "\tzend_atomic_bool_store_ex(&EG(timed_out), false);\n" + "\t/* A responsive VM timeout follows POSIX zend_timeout(): disarm before shutdown. */\n" "\tzend_set_timeout_ex(0, 1);\n" "# if defined(__wasm32__) || defined(__wasm64__)\n" - "\tif (EG(hard_timeout) > 0) {\n" - "\t\tzend_wasm_arm_hard_timeout(EG(hard_timeout));\n" - "\t\t__wasm_posix_vm_interrupt_after(&EG(timed_out), &EG(vm_interrupt), EG(hard_timeout));\n" + "\t/* Give shutdown execution a fresh ordinary timeout phase. */\n" + "\tif (EG(timeout_seconds) > 0) {\n" + "\t\tzend_wasm_record_timeout_deadline(EG(timeout_seconds));\n" + "\t\t__wasm_posix_vm_interrupt_after(&EG(timed_out), &EG(vm_interrupt), EG(timeout_seconds));\n" "\t}\n" "# endif\n" "#endif\n\n" diff --git a/packages/registry/php/build.toml b/packages/registry/php/build.toml index 6fad09fa0f..10e35d9d81 100644 --- a/packages/registry/php/build.toml +++ b/packages/registry/php/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/php/build-php.sh" repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 8 +revision = 10 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" From 74a5765574f7f4a2243f77d976a762602c54771c Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 07:49:40 -0400 Subject: [PATCH 27/29] fix(netcat): avoid unsupported abortive close on Kandelo GNU netcat treats reset-style SO_LINGER as an optimization. Kandelo truthfully rejects that mode until every transport can model it, so the wasm package retains ordinary FIN close semantics instead. Make source builds resolver-safe at the same time: activate the worktree SDK, verify resolver-provided source identity, keep scratch state under the resolver work directory, and install resolver builds only into their declared output directory. --- packages/registry/netcat/build-netcat.sh | 72 ++++++++++++++++--- packages/registry/netcat/build.toml | 2 +- .../patches/disable-abortive-linger.patch | 33 +++++++++ 3 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 packages/registry/netcat/patches/disable-abortive-linger.patch diff --git a/packages/registry/netcat/build-netcat.sh b/packages/registry/netcat/build-netcat.sh index 1ae2382684..378791c15d 100755 --- a/packages/registry/netcat/build-netcat.sh +++ b/packages/registry/netcat/build-netcat.sh @@ -3,15 +3,28 @@ set -euo pipefail # Build GNU Netcat 0.7.1 for wasm32-posix-kernel. -NETCAT_VERSION="${NETCAT_VERSION:-0.7.1}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -SRC_DIR="$SCRIPT_DIR/netcat-src" -BIN_DIR="$SCRIPT_DIR/bin" +NETCAT_VERSION="${WASM_POSIX_DEP_VERSION:-${NETCAT_VERSION:-0.7.1}}" +SOURCE_URL="${WASM_POSIX_DEP_SOURCE_URL:-https://downloads.sourceforge.net/project/netcat/netcat/${NETCAT_VERSION}/netcat-${NETCAT_VERSION}.tar.gz}" +SOURCE_SHA256="${WASM_POSIX_DEP_SOURCE_SHA256:-30719c9a4ffbcf15676b8f528233ccc54ee6cba96cb4590975f5fd60c68a066f}" +TARGET_ARCH="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" +WORK_DIR="${WASM_POSIX_DEP_WORK_DIR:-$SCRIPT_DIR}" +SRC_DIR="$WORK_DIR/netcat-src" +BIN_DIR="$WORK_DIR/bin" SYSROOT="${WASM_POSIX_SYSROOT:-$REPO_ROOT/sysroot}" +# Worktree-local SDK on PATH (no global npm link required). +# shellcheck source=/dev/null +source "$REPO_ROOT/sdk/activate.sh" + +if [ "$TARGET_ARCH" != "wasm32" ]; then + echo "ERROR: GNU Netcat is currently packaged for wasm32 only, got $TARGET_ARCH" >&2 + exit 2 +fi + if ! command -v wasm32posix-cc &>/dev/null; then - echo "ERROR: wasm32posix-cc not found. Run 'npm link' in sdk/ first." >&2 + echo "ERROR: wasm32posix-cc not found after sourcing sdk/activate.sh." >&2 exit 1 fi @@ -23,14 +36,26 @@ fi export WASM_POSIX_SYSROOT="$SYSROOT" export WASM_POSIX_GLUE_DIR="$REPO_ROOT/libc/glue" +SOURCE_MARKER="$SRC_DIR/.kandelo-netcat-source" +expected_source_marker="$(printf '%s\n%s\n%s' "$NETCAT_VERSION" "$SOURCE_URL" "$SOURCE_SHA256")" +if [ -d "$SRC_DIR" ] && [ "$(cat "$SOURCE_MARKER" 2>/dev/null || true)" != "$expected_source_marker" ]; then + echo "==> Existing GNU Netcat source does not match requested version/source; cleaning..." + rm -rf "$SRC_DIR" "$BIN_DIR" +fi + if [ ! -d "$SRC_DIR" ]; then echo "==> Downloading GNU Netcat $NETCAT_VERSION..." + DOWNLOAD_DIR="$(mktemp -d "${TMPDIR:-/tmp}/kandelo-netcat-src.XXXXXX")" + trap 'rm -rf "$DOWNLOAD_DIR"' EXIT TARBALL="netcat-${NETCAT_VERSION}.tar.gz" - URL="https://downloads.sourceforge.net/project/netcat/netcat/${NETCAT_VERSION}/${TARBALL}" - curl --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors -fsSL "$URL" -o "/tmp/$TARBALL" + curl --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors -fsSL "$SOURCE_URL" -o "$DOWNLOAD_DIR/$TARBALL" + echo "==> Verifying source sha256..." + echo "$SOURCE_SHA256 $DOWNLOAD_DIR/$TARBALL" | shasum -a 256 -c - mkdir -p "$SRC_DIR" - tar xzf "/tmp/$TARBALL" -C "$SRC_DIR" --strip-components=1 - rm "/tmp/$TARBALL" + tar xzf "$DOWNLOAD_DIR/$TARBALL" -C "$SRC_DIR" --strip-components=1 + printf '%s\n' "$expected_source_marker" > "$SOURCE_MARKER" + trap - EXIT + rm -rf "$DOWNLOAD_DIR" fi cd "$SRC_DIR" @@ -40,6 +65,7 @@ PATCH_SET=( "listen-success-exit.patch" "udp-listen-single-socket.patch" "disable-pktinfo.patch" + "disable-abortive-linger.patch" ) echo "==> Verifying Kandelo netcat portability patches..." for patch_name in "${PATCH_SET[@]}"; do @@ -76,6 +102,11 @@ if ! grep -q "/\\* # define USE_PKTINFO \\*/" src/netcat.h; then exit 1 fi +if ! grep -q "Kandelo cannot yet model abortive SO_LINGER" src/network.c; then + echo "ERROR: disable-abortive-linger.patch is missing from src/network.c" >&2 + exit 1 +fi + if [ ! -f Makefile ]; then echo "==> Configuring GNU Netcat for wasm32..." export ac_cv_func_malloc_0_nonnull=yes @@ -108,13 +139,32 @@ fi echo "==> Applying fork instrumentation metadata..." FORK_INSTRUMENT="$REPO_ROOT/scripts/run-wasm-fork-instrument.sh" -"$FORK_INSTRUMENT" "$NETCAT_BIN" -o "$NETCAT_BIN.instr" +(cd "$REPO_ROOT" && "$FORK_INSTRUMENT" "$NETCAT_BIN" -o "$NETCAT_BIN.instr") mv "$NETCAT_BIN.instr" "$NETCAT_BIN" mkdir -p "$BIN_DIR" cp "$NETCAT_BIN" "$BIN_DIR/nc.wasm" -source "$REPO_ROOT/scripts/install-local-binary.sh" -install_local_binary netcat "$BIN_DIR/nc.wasm" +if [ -n "${WASM_POSIX_DEP_OUT_DIR:-}" ]; then + # Resolver builds must publish only into the resolver-owned output + # directory. Apply the same artifact guards as install_local_binary without + # writing a local-binaries override into the source worktree. + # shellcheck source=/dev/null + source "$REPO_ROOT/scripts/wasm-artifact-guards.sh" + if ! wasm_is_binary "$BIN_DIR/nc.wasm"; then + echo "ERROR: refusing non-Wasm netcat artifact: $BIN_DIR/nc.wasm" >&2 + exit 1 + fi + wasm_require_no_legacy_asyncify "$BIN_DIR/nc.wasm" + wasm_require_fork_instrumentation_if_needed "$BIN_DIR/nc.wasm" + mkdir -p "$WASM_POSIX_DEP_OUT_DIR" + cp "$BIN_DIR/nc.wasm" "$WASM_POSIX_DEP_OUT_DIR/nc.wasm" + echo " installed $WASM_POSIX_DEP_OUT_DIR/nc.wasm (resolver scratch)" +else + # Direct developer builds retain the normal local resolver override. + # shellcheck source=/dev/null + source "$REPO_ROOT/scripts/install-local-binary.sh" + install_local_binary netcat "$BIN_DIR/nc.wasm" +fi ls -lh "$BIN_DIR/nc.wasm" diff --git a/packages/registry/netcat/build.toml b/packages/registry/netcat/build.toml index e19e4a7fa0..e910c2d7c7 100644 --- a/packages/registry/netcat/build.toml +++ b/packages/registry/netcat/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/netcat/build-netcat.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "2e6293a50ccf996b1a434aa701057b862a46c587" -revision = 1 +revision = 2 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/netcat/patches/disable-abortive-linger.patch b/packages/registry/netcat/patches/disable-abortive-linger.patch new file mode 100644 index 0000000000..855af9bfe7 --- /dev/null +++ b/packages/registry/netcat/patches/disable-abortive-linger.patch @@ -0,0 +1,33 @@ +diff --git a/src/network.c b/src/network.c +index 23b2d2c..62e1f51 100644 +--- a/src/network.c ++++ b/src/network.c +@@ -351,6 +351,8 @@ int netcat_socket_new(int domain, int type) + { + int sock, ret, sockopt; ++#if !defined(__wasm32__) + struct linger fix_ling; ++#endif + + sock = socket(domain, type, 0); + if (sock < 0) +@@ -359,5 +361,11 @@ int netcat_socket_new(int domain, int type) + + /* don't leave the socket in a TIME_WAIT state if we close the connection */ ++#if defined(__wasm32__) ++ /* Kandelo cannot yet model abortive SO_LINGER coherently across its ++ in-kernel, Node, and browser transports. The platform truthfully rejects ++ the enabled option, so retain ordinary FIN close semantics here. */ ++ ret = 0; ++#else + fix_ling.l_onoff = 1; + fix_ling.l_linger = 0; + ret = setsockopt(sock, SOL_SOCKET, SO_LINGER, &fix_ling, sizeof(fix_ling)); +@@ -367,6 +375,7 @@ int netcat_socket_new(int domain, int type) + close(sock); /* anyway the socket was created */ + return -2; + } ++#endif + + /* fix the socket options */ + sockopt = 1; From 15e0e748cbbf1b8dac17c6bcc3cf7f958f22d128 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 08:15:03 -0400 Subject: [PATCH 28/29] fix: support Linux coarse clock requests Map CLOCK_REALTIME_COARSE and CLOCK_MONOTONIC_COARSE to Kandelo's corresponding host clock domains. The hosts do not expose distinct coarse sources, but returning EINVAL causes libc consumers such as MariaDB to retry before startup. Cover clock_gettime and clock_getres and document the fallback. --- crates/kernel/src/syscalls.rs | 32 ++++++++++++++++++++++++++++++-- crates/shared/src/lib.rs | 2 ++ docs/posix-status.md | 2 +- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/crates/kernel/src/syscalls.rs b/crates/kernel/src/syscalls.rs index c7ebc254ce..0c9a475119 100644 --- a/crates/kernel/src/syscalls.rs +++ b/crates/kernel/src/syscalls.rs @@ -5546,7 +5546,6 @@ pub fn sys_exit(proc: &mut Process, host: &mut dyn HostIO, status: i32) { proc.exit_signal = 0; } -/// Get the current time from the specified clock. fn is_encoded_process_cpu_clock_id(clock_id: u32) -> bool { // musl/Linux encode process CPU clocks as (-pid-1)*8 + 2. Real encoded // IDs are negative clockid_t values; checking only the low three bits @@ -5561,6 +5560,13 @@ fn host_clock_id(clock_id: u32) -> Result { match clock_id { CLOCK_REALTIME | CLOCK_MONOTONIC | CLOCK_PROCESS_CPUTIME_ID | CLOCK_THREAD_CPUTIME_ID | CLOCK_BOOTTIME => Ok(clock_id), + // Linux exposes coarse variants to libc consumers such as MariaDB. + // Kandelo's hosts do not maintain separate coarse clock sources, so + // preserve the clock domain while using the corresponding canonical + // source. Returning EINVAL here leaves MariaDB spinning before its + // first filesystem operation. + CLOCK_REALTIME_COARSE => Ok(CLOCK_REALTIME), + CLOCK_MONOTONIC_COARSE => Ok(CLOCK_MONOTONIC), // clock_getcpuclockid() encodes a process clock as (-pid-1)*8 + 2. // Kandelo does not yet account CPU time per process, so preserve the // documented elapsed-time approximation without letting the host @@ -5599,7 +5605,12 @@ pub fn sys_nanosleep( pub fn sys_clock_getres(_proc: &Process, clock_id: u32) -> Result { use wasm_posix_shared::clock::*; match clock_id { - CLOCK_REALTIME | CLOCK_MONOTONIC | CLOCK_PROCESS_CPUTIME_ID | CLOCK_THREAD_CPUTIME_ID + CLOCK_REALTIME + | CLOCK_MONOTONIC + | CLOCK_PROCESS_CPUTIME_ID + | CLOCK_THREAD_CPUTIME_ID + | CLOCK_REALTIME_COARSE + | CLOCK_MONOTONIC_COARSE | CLOCK_BOOTTIME => Ok(WasmTimespec { tv_sec: 0, tv_nsec: 1_000_000, @@ -15861,6 +15872,23 @@ mod tests { assert!(sys_clock_gettime(&proc, &mut host, (-2_i32 * 8 + 2) as u32).is_ok()); } + #[test] + fn test_clock_calls_map_linux_coarse_clocks_to_supported_sources() { + use wasm_posix_shared::clock::{ + CLOCK_MONOTONIC, CLOCK_MONOTONIC_COARSE, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, + }; + + let proc = Process::new(1); + let mut host = MockHostIO::new(); + + assert_eq!(host_clock_id(CLOCK_REALTIME_COARSE), Ok(CLOCK_REALTIME)); + assert_eq!(host_clock_id(CLOCK_MONOTONIC_COARSE), Ok(CLOCK_MONOTONIC),); + for clock_id in [CLOCK_REALTIME_COARSE, CLOCK_MONOTONIC_COARSE] { + assert!(sys_clock_gettime(&proc, &mut host, clock_id).is_ok()); + assert!(sys_clock_getres(&proc, clock_id).is_ok()); + } + } + #[test] fn test_clock_gettime_rejects_unknown_clock() { let proc = Process::new(1); diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs index aa888ffefc..510ae8962e 100644 --- a/crates/shared/src/lib.rs +++ b/crates/shared/src/lib.rs @@ -897,6 +897,8 @@ pub mod clock { pub const CLOCK_MONOTONIC: u32 = 1; pub const CLOCK_PROCESS_CPUTIME_ID: u32 = 2; pub const CLOCK_THREAD_CPUTIME_ID: u32 = 3; + pub const CLOCK_REALTIME_COARSE: u32 = 5; + pub const CLOCK_MONOTONIC_COARSE: u32 = 6; pub const CLOCK_BOOTTIME: u32 = 7; } diff --git a/docs/posix-status.md b/docs/posix-status.md index 8e2e0148a8..245aee249d 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -259,7 +259,7 @@ shortcuts. |----------|--------|-------| | `time()` | Full | Wrapper around clock_gettime(CLOCK_REALTIME). Returns seconds since epoch. | | `gettimeofday()` | Full | Wrapper around clock_gettime(CLOCK_REALTIME). Returns (sec, usec) pair. | -| `clock_gettime()` | Partial | Host-delegated. `CLOCK_REALTIME` and `CLOCK_MONOTONIC` are supported. `CLOCK_BOOTTIME` is a monotonic-equivalent fallback because Kandelo hosts do not expose suspend accounting. The process/thread CPU clocks currently report elapsed monotonic time rather than authoritative CPU usage. Linux-style encoded process CPU clock IDs must be negative; malformed positive encodings are rejected with `EINVAL` (which musl's `clock_getcpuclockid()` maps to `ESRCH`). Node.js uses `Date.now()` and `process.hrtime.bigint()`; browsers use `Date.now()` and `performance.now()`. | +| `clock_gettime()` | Partial | Host-delegated. `CLOCK_REALTIME` and `CLOCK_MONOTONIC` are supported. Linux `CLOCK_REALTIME_COARSE` and `CLOCK_MONOTONIC_COARSE` requests use the corresponding host clock as an equivalent fallback because Kandelo hosts do not expose separate coarse sources. `CLOCK_BOOTTIME` is a monotonic-equivalent fallback because Kandelo hosts do not expose suspend accounting. The process/thread CPU clocks currently report elapsed monotonic time rather than authoritative CPU usage. Linux-style encoded process CPU clock IDs must be negative; malformed positive encodings are rejected with `EINVAL` (which musl's `clock_getcpuclockid()` maps to `ESRCH`). Node.js uses `Date.now()` and `process.hrtime.bigint()`; browsers use `Date.now()` and `performance.now()`. | | `nanosleep()` | Partial | Host-delegated. Node.js uses Atomics.wait with timeout. Browser support requires a worker context that can block with Atomics.wait. Validates tv_sec >= 0 and tv_nsec < 1e9. | | `usleep()` | Full | Converts microseconds to sec+nsec, delegates to host_nanosleep. | | `clock_settime()` | Stub | Returns EPERM. Cannot set system clock from Wasm. | From bb5a0b19451c39d7ae7d1bb0b15aa3b0cf4c468f Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 11 Jul 2026 08:15:08 -0400 Subject: [PATCH 29/29] docs: describe VM interrupt flag stores accurately Each timeout flag byte is written with an atomic store; the pair is not one atomic transaction. --- docs/architecture.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 4afeea25e4..4a4642917b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -135,11 +135,11 @@ kernel hook. The process worker forwards each arm or cancellation request to its host's dedicated kernel worker. The kernel worker owns the JavaScript timer because a process worker executing a CPU-bound Wasm loop cannot service its own event -loop. At the monotonic deadline the host atomically sets both bytes in the -process's shared memory. Timer entries retain the exact process-generation -object as well as the PID, so exec, exit, or PID reuse cannot redirect a stale -callback into a replacement process. Deadlines beyond JavaScript's signed -32-bit timer range are scheduled in bounded chunks. +loop. At the monotonic deadline the host sets each flag byte with an atomic +store in the process's shared memory. Timer entries retain the exact +process-generation object as well as the PID, so exec, exit, or PID reuse +cannot redirect a stale callback into a replacement process. Deadlines beyond +JavaScript's signed 32-bit timer range are scheduled in bounded chunks. The imported function and its pointer-width-specific signature are part of the guest/host ABI. A package that starts importing it must raise its kernel