diff --git a/.gitignore b/.gitignore index 1fa05ac98a..91949fe0cf 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ registry/native/c/build/ registry/native/c/vendor/ registry/native/c/libs/ registry/native/c/.cache/ +toolchain/c/.cache/ registry/native/c/sysroot/ registry/software/*/wasm/ software/*/wasm/ diff --git a/README.md b/README.md index 2d1e80bfb8..bc241e805c 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,7 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent | `@agentos-software/tar` | tar | GNU tar archiver | rust | - | - | | `@agentos-software/tree` | tree | tree directory listing | rust | - | - | | `@agentos-software/unzip` | unzip | unzip archive extraction | c | - | - | +| `@agentos-software/wget` | wget | GNU Wget file downloader | c | - | - | | `@agentos-software/yq` | yq | yq YAML/JSON processor | rust | - | - | | `@agentos-software/zip` | zip | zip archive creation | c | - | - | @@ -174,7 +175,7 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent |---------|-------------|----------| | `@agentos-software/build-essential` | Build-essential WASM command set (standard + make + git + curl) | standard, make, git, curl | | `@agentos-software/common` | Common WASM command set (coreutils + sed + grep + gawk + findutils + diffutils + tar + gzip) | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip | -| `@agentos-software/everything` | All available WASM command packages in a single bundle | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip, curl, zip, unzip, jq, ripgrep, fd, tree, file, yq, codex-cli | +| `@agentos-software/everything` | All available WASM command packages in a single bundle | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip, curl, wget, zip, unzip, jq, ripgrep, fd, tree, file, yq, codex-cli | ## License diff --git a/crates/execution/assets/runners/wasm-runner.mjs b/crates/execution/assets/runners/wasm-runner.mjs index 6c4d85887b..47e83c2178 100644 --- a/crates/execution/assets/runners/wasm-runner.mjs +++ b/crates/execution/assets/runners/wasm-runner.mjs @@ -1776,6 +1776,91 @@ function writeBytesToGuestIovs(iovs, iovsLen, bytes) { return written >>> 0; } +function guestIovByteLength(iovs, iovsLen) { + if (!(instanceMemory instanceof WebAssembly.Memory)) { + throw new Error('WebAssembly memory is not available'); + } + + const view = new DataView(instanceMemory.buffer); + let total = 0; + for (let index = 0; index < (Number(iovsLen) >>> 0); index += 1) { + const entryOffset = (Number(iovs) >>> 0) + index * 8; + total += view.getUint32(entryOffset + 4, true); + } + return total >>> 0; +} + +function readHostNetSocketToGuestIovs(socket, iovs, iovsLen, nreadPtr) { + try { + const requestedLength = guestIovByteLength(iovs, iovsLen); + if (requestedLength === 0) { + return writeGuestUint32(nreadPtr, 0); + } + + if (socket.nonblock) { + let queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.lastError) return WASI_ERRNO_FAULT; + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + pollHostNetSocket(socket, 0); + queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + return WASI_ERRNO_AGAIN; + } + + const deadline = + socket.recvTimeoutMs == null ? null : Date.now() + Math.max(0, socket.recvTimeoutMs); + while (true) { + const queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.lastError) return WASI_ERRNO_FAULT; + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + + const pollWaitMs = + deadline == null ? 50 : Math.max(0, Math.min(50, deadline - Date.now())); + if (deadline != null && pollWaitMs === 0) { + return WASI_ERRNO_AGAIN; + } + pollHostNetSocket(socket, pollWaitMs); + if (deadline != null && Date.now() >= deadline) { + return WASI_ERRNO_AGAIN; + } + } + } catch { + return WASI_ERRNO_FAULT; + } +} + +function writeHostNetSocketFromGuestIovs(socket, iovs, iovsLen, nwrittenPtr) { + if (!socket?.socketId || socket.closed) { + return WASI_ERRNO_BADF; + } + + try { + const bytes = collectGuestIovBytes(iovs, iovsLen); + if (bytes.length === 0) { + return writeGuestUint32(nwrittenPtr, 0); + } + const written = Number(callSyncRpc('net.write', [socket.socketId, bytes])) >>> 0; + return writeGuestUint32(nwrittenPtr, written); + } catch { + return WASI_ERRNO_FAULT; + } +} + function dequeuePipeBytes(pipe, maxBytes) { const requested = Math.max(0, Number(maxBytes) >>> 0); if (requested === 0 || pipe.chunks.length === 0) { @@ -2700,8 +2785,9 @@ function callSyncRpc(method, args = []) { } const hostNetSockets = new Map(); -let nextHostNetSocketFd = 0x40000000; +let nextHostNetSocketFd = 4096; const HOST_NET_TIMEOUT_SENTINEL = '__agentos_net_timeout__'; +const HOST_NET_MSG_PEEK = 0x0001; function getHostNetSocket(fd) { return hostNetSockets.get(Number(fd) >>> 0) ?? null; @@ -2732,6 +2818,76 @@ function dequeueHostNetBytes(socket, maxBytes) { return Buffer.concat(parts); } +function peekHostNetBytes(socket, maxBytes) { + const requested = Math.max(0, Number(maxBytes) >>> 0); + if (requested === 0 || socket.readChunks.length === 0) { + return Buffer.alloc(0); + } + + const parts = []; + let remaining = requested; + for (const chunk of socket.readChunks) { + if (remaining === 0) break; + const chunkLength = Math.min(chunk.length, remaining); + parts.push(chunk.subarray(0, chunkLength)); + remaining -= chunkLength; + } + + return Buffer.concat(parts); +} + +function decodeHostNetSocketReadResult(result) { + if (result == null) { + return { kind: 'end' }; + } + + if (result === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + + if (typeof result === 'string') { + if (result === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + return { kind: 'data', bytes: Buffer.from(result, 'base64') }; + } + + const decoded = decodeSyncRpcValue(result); + if (Buffer.isBuffer(decoded)) { + return { kind: 'data', bytes: decoded }; + } + if (decoded == null) { + return { kind: 'end' }; + } + if (decoded === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + return { kind: 'timeout' }; +} + +function readReadyHostNetSocket(socket) { + if (!socket?.socketId || socket.closed) { + socket.readableEnded = true; + return null; + } + + const result = decodeHostNetSocketReadResult( + callSyncRpc('net.socket_read', [socket.socketId]), + ); + if (result.kind === 'data') { + if (result.bytes.length > 0) { + socket.readChunks.push(Buffer.from(result.bytes)); + } + return result; + } + if (result.kind === 'end') { + socket.readableEnded = true; + socket.closed = true; + socket.socketId = null; + } + return result; +} + function pollHostNetSocket(socket, waitMs) { if (!socket?.socketId || socket.closed) { return null; @@ -2766,6 +2922,20 @@ function pollHostNetSocket(socket, waitMs) { return event; } + if (event.readable === true || (Number(event.revents) & 0x001) !== 0) { + return readReadyHostNetSocket(socket); + } + + if (event.hangup === true) { + socket.readableEnded = true; + return event; + } + + if (event.error === true) { + socket.lastError = 'socket error'; + return event; + } + return event; } @@ -3493,15 +3663,14 @@ const hostNetImport = { } try { - if ((Number(flags) >>> 0) !== 0) { - // Non-zero recv flags are currently ignored in the WASM host_net shim. - } + const recvFlags = Number(flags) >>> 0; + const peek = (recvFlags & HOST_NET_MSG_PEEK) !== 0; // Non-blocking sockets (O_NONBLOCK via net_set_nonblock, used by libxcb's poll_for_*): // pull whatever is queued, do ONE short readiness probe, and return EAGAIN if still empty // instead of blocking. libxcb assumes its "poll" reads never block on an empty socket. if (socket.nonblock) { - let queued = dequeueHostNetBytes(socket, bufLen); + let queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -3510,7 +3679,7 @@ const hostNetImport = { return writeGuestUint32(retReceivedPtr, 0); } pollHostNetSocket(socket, 0); - queued = dequeueHostNetBytes(socket, bufLen); + queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -3523,7 +3692,7 @@ const hostNetImport = { const deadline = socket.recvTimeoutMs == null ? null : Date.now() + Math.max(0, socket.recvTimeoutMs); while (true) { - const queued = dequeueHostNetBytes(socket, bufLen); + const queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -4868,6 +5037,11 @@ const KERNEL_POLLHUP = 0x0010; wasiImport.fd_read = (fd, iovs, iovsLen, nreadPtr) => { const numericFd = Number(fd) >>> 0; + const hostNetSocket = getHostNetSocket(numericFd); + if (hostNetSocket) { + return readHostNetSocketToGuestIovs(hostNetSocket, iovs, iovsLen, nreadPtr); + } + const handle = __agentOSWasiMeasurePhase('fd_read', 'lookup_handle', () => lookupFdHandle(numericFd) ); @@ -5522,10 +5696,15 @@ wasiImport.fd_prestat_dir_name = (fd, pathPtr, pathLen) => { }; wasiImport.fd_write = (fd, iovs, iovsLen, nwrittenPtr) => { + const numericFd = Number(fd) >>> 0; + const hostNetSocket = getHostNetSocket(numericFd); + if (hostNetSocket) { + return writeHostNetSocketFromGuestIovs(hostNetSocket, iovs, iovsLen, nwrittenPtr); + } + const handle = __agentOSWasiMeasurePhase('fd_write', 'lookup_handle', () => lookupFdHandle(fd) ); - const numericFd = Number(fd) >>> 0; if (handle?.kind === 'pipe-write') { try { const bytes = __agentOSWasiMeasurePhase('fd_write', 'guest_iov_collect', () => diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index 64d9c95886..86af628d17 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -100,16 +100,16 @@ actual backing: ### ❌ CUSTOM WE BUILT — flag & replace with a real/established impl | Command | Status | What it actually is | Replace with | |---|---|---|---| -| **curl** | TODO | our custom driver over a libcurl fork | real `curl` CLI (upstream `src/tool_*.c`) | -| **wget** | TODO (retry) | our 174-line `wget.c` (dropped) | real GNU Wget vs our sysroot — stub `getrlimit`/`getgroups`, then build | +| **curl** | DONE | our custom driver over a libcurl fork | real `curl` CLI (upstream `src/tool_*.c`) | +| **wget** | DONE | our 174-line `wget.c` (dropped) | real GNU Wget vs our sysroot — stub `getrlimit`/`getgroups`, then build | | **http-get** | TODO | our 95-line `http_get.c` | drop, or a real tool | | **git** | TODO | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | | **fd** | TODO | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | | **findutils** (`find`,`xargs`) | TODO | our hand-rolled on `regex`/shims | real GNU findutils, or `uutils/findutils` | | **tree** | TODO | our hand-rolled, zero deps | real `tree`, or an established one | | **grep** | TODO | our `secureexec-grep` on raw `regex` (**not** an established grep pkg) | **real GNU grep**, or a popular established grep (ripgrep's `grep` crates) | -| **zip** | TODO | our 203-line `zip.c` over zlib/minizip (not Info-ZIP) | real Info-ZIP, or an established lib's CLI | -| **unzip** | TODO | our 669-line `unzip.c` over zlib/minizip | real Info-ZIP unzip | +| **zip** | DONE | our 203-line `zip.c` over zlib/minizip (not Info-ZIP) | real Info-ZIP, or an established lib's CLI | +| **unzip** | DONE | our 669-line `unzip.c` over zlib/minizip | real Info-ZIP unzip | | **sqlite3 CLI** | TODO | our 558-line `sqlite3_cli.c` (engine is real SQLite; the shell is ours) | real SQLite `shell.c` (its official CLI) | | **vix** | DONE | from-scratch source-less drop-zone binary | deleted; real `vim` covers the editor slot | @@ -130,6 +130,69 @@ impl *before* their tests are written, so the tests validate real behavior. **Decisions (settled):** git → **real git** (not gitoxide). grep → **real GNU grep**, or a popular established grep if the real one won't build. +**git — where the issues are (assessment):** +- **LICENSE — RESOLVED, not a blocker.** Each command ships as its **own published + npm package**, so a GPL-2.0 git binary in `@agentos-software/git` is **mere + aggregation** — it does not affect the Apache-2.0 licensing of agentOS or any + other package. Ship the git package under GPL-2.0 (offer its source) and we're + compliant. This **supersedes** the clean-room-reimpl rationale in + `toolchain/std-patches/git/README.md` ("cannot be vendored due to license + restrictions"): that premise no longer holds — go with **real git** (upstream C) + and update/remove that README. (gitoxide stays ruled out.) +- **Technical WASI issues if we do build real git** (from that README + git's own + build knobs), easiest → hardest: + - `mmap` (packfiles/index) → build `NO_MMAP=1` (malloc+read). Fine. + - signals (SIGPIPE/SIGCHLD) → build without; WASI has none. Fine. + - threads (index-pack/pack-objects) → `NO_PTHREADS`; single-threaded, slower. Fine. + - `fork()`+`exec()` in `run_command.c` (hooks, filters, remote helpers) → route to + `posix_spawn` via the `wasi-spawn` broker (spawn IS supported — same fix as wget). + - **Network transport (clone/fetch/push) — the hard part.** Smart-HTTP needs the + `git-remote-https` **helper subprocess** + libcurl; `git://` needs raw sockets; + ssh needs an `ssh` subprocess. Each helper must itself exist as a module. + - symlink checkout → `core.symlinks=false` fallback (WASI symlink support is + partial); local time → UTC like elsewhere. +- **Bottom line:** license is a non-issue (separate published package = mere + aggregation). **local** git (init/add/commit/log/diff/branch/merge/status/ + checkout) is very achievable; **remote** git (clone/fetch/push over + smart-HTTP/ssh) is the real effort. Proceed with real git. + +**Replacement findings — what each remaining ❌ tool will take (investigated):** + +The recurring wall is **never a syscall** — it's one of two known, already-solved +patterns: **(a) no threads** on `wasm32-wasip1` → serial patch like +`toolchain/std-patches/crates/uu_sort/0001-wasi-serial-sort.patch` (hits real fd & +ripgrep crates); **(b) gnulib `getrlimit`/`getgroups`** → sysroot stubs, already +documented for wget (item #8) (hits GNU grep/findutils). Subprocess spawn already +works (`wasi-spawn` broker), so `xargs` is not a blocker. + +- **sqlite3 CLI — trivial. Start here.** Engine is already the real amalgamation + (`libs/sqlite3/sqlite3.c`); the official `shell.c` **ships in the same fetched + zip**. Add `shell.c` to the `cp` in `toolchain/c/Makefile:174` and compile it in + place of `sqlite3_cli.c` (build rule ~498), reusing the existing SQLite mem/`-Os` + flags. Same real engine, real upstream shell. +- **http-get — drop, don't port.** `http_get.c` is a 95-line raw-socket **loopback + test client**, not an HTTP fetcher; real curl covers GET. But it's imported by + `packages/shell` and is the client in cross-runtime network tests — migrate those + dependents first, then drop the command. +- **tree — easy.** Real `tree` (Steve Baker) is a tiny plain-`Makefile` C program; + compile its `.c` directly against the sysroot. Pure readdir/stat — no + sockets/threads/spawn. +- **fd — moderate.** Swap `cmd-fd` to depend on real `fd-find` (like + coreutils→`uu_*`; `fd-lock` is already patched). Only real issue: the parallel + `ignore`/crossbeam walker needs the **serial-thread patch** (uu_sort pattern). +- **grep — moderate.** Decision is real GNU grep (gnulib cascade → sysroot stubs) + or ripgrep's `grep-*` crates (threads → serial patch). NB the current + `secureexec-grep` also backs `rg`, so the shipped "ripgrep" is custom too. +- **zip / unzip — moderate.** Real **Info-ZIP** source (fetch+pin like zlib/sqlite); + zlib is already vendored. Filesystem + `isatty`/`utime`/`chmod`/perms stubs; no + sockets/threads/spawn. Friction is Info-ZIP's crufty build, not syscalls. +- **findutils — moderate→hard.** `find` is fs+regex (easy); `xargs` spawn already + works. **uutils/findutils** (Rust) avoids gnulib; **GNU findutils** (C) hits the + same gnulib cascade as wget. Prefer uutils unless GNU parity is required. + +Ranked easiest→hardest: **sqlite3-CLI · http-get (drop) · tree · fd · grep · zip · +unzip · findutils.** + ## Status tracking (how the driver reports progress in this doc) Update this doc as you go — it is the single source of truth for status. For each @@ -227,6 +290,13 @@ so a reader sees the whole board at a glance. build/protocol checks pass in `2026-07-08T00-41-51-0700-item6-runtime-core-build-tls-flags.txt`. - **rev:** `oxoqrwvk` — `fix(curl): build the real curl CLI for WASI` +- **Note (how well it works):** it *is* the real curl CLI (`src/tool_main.c`) plus a + custom `vtls/wasi_tls.c` backend (`USE_WASI_TLS`) — HTTPS runs through the host + TLS bridge, not OpenSSL. Real HTTP(S) `GET/POST/-I/-D/-L/-u/-F/-o/-O/-w/-K` all + work because it's genuine curl. Known gaps from the trimmed `./configure`: + `--compressed`/gzip response decode (`--without-zlib`), brotli/zstd, `libpsl` + cookie-suffix checks, LDAP, and no CA bundle (cert trust is whatever `wasi_tls` + enforces). Those are the 5 skipped tests. Verdict: solid for real HTTP(S). ### 7. zip / unzip — hostile-archive hardening cases fail (3 each) — DONE - **Broken:** fallback parser doesn't reject a wrapping local offset, doesn't skip @@ -246,31 +316,26 @@ so a reader sees the whole board at a glance. ## P2 — Build / compile failures -### 8. wget — TODO (retry against our own sysroot) -- **Objective:** ship real GNU Wget as the `wget` command (real upstream tool, - patched as needed), proven by a real e2e download test. The old package was a - custom 174-line `wget.c` wrapper and was dropped — it must be restored as the - real tool, not re-added as a shim. -- **Prior attempt (bailed too early):** GNU Wget 1.24.5 configured for - `wasm32-wasip1`, HTTP-only (`--without-ssl --without-zlib --without-libpsl - --disable-iri`, auth/thread options off) against the patched sysroot. It needed - WASI patches for `inet_addr`, `O_BINARY`, process-group terminal checks, - `spawn.h`/`--use-askpass`, interactive `getpass`, gnulib `NSIG`, `F_DUPFD`, and - `flock` — all fine — then stopped at gnulib wanting `getrlimit`/`RLIMIT_NOFILE` - and an incompatible `getgroups`. -- **Why that stop was wrong:** the sysroot is ours (see CLAUDE.md → Software Build - (WASM Toolchain)). `getrlimit` can return a fixed `RLIMIT_NOFILE`; `getgroups` - can return the single group. Those are a few-line stub in the patched - libc/host-import layer, exactly like the spawn/fd/user-group imports already - added. "WASI lacks it" is not a wall here. -- **Retry plan:** (1) add `getrlimit`/`RLIMIT_NOFILE` and a compatible `getgroups` - to the sysroot (stub is acceptable); (2) clear the remaining gnulib cascade the - same way, one patch at a time; (3) patch Wget's own source only if a fix - genuinely cannot live in the libc layer; (4) restore the `wget` package/command - and prove it with a real e2e download test. Only mark `NOT POSSIBLE (WASI)` - again if a concrete, documented wall remains after that. -- **Note:** real upstream `curl` (#6) already covers downloads, so wget is not - urgent — but it should be retried as the real tool, not left dropped. +### 8. wget — DONE +- **Resolved:** the `wget` command is real upstream GNU Wget 1.24.5 built for + `wasm32-wasip1`, HTTP-only for now (`--without-ssl --without-zlib + --without-libpsl --disable-iri`) against the patched AgentOS C sysroot. The old + custom 174-line wrapper stays removed. +- **Sysroot/runtime fixes:** Wget builds without a Wget-source WASI fork by adding + the missing POSIX surface one layer down: process/terminal headers including + `spawn.h`, signal/process/timezone compatibility, overrideable `FD_SETSIZE`, + Wget-only `_POSIX_TIMERS` overlay, POSIX socket `read`/`write` routing through + `host_net`, low host-net fds, and `MSG_PEEK` queue preservation in the WASM + runner. Configure is seeded so gnulib trusts the sysroot `select` instead of + replacing it with a host-net-incompatible fallback. +- **Proof:** focused basename download passes in + `2026-07-08T04-33-31-0700-item8-wget-vitest-focused-clean-msg-peek.log`; full + Wget e2e suite passes 5/5 in + `2026-07-08T04-33-41-0700-item8-wget-vitest-full-clean-msg-peek.log`. Final + runner syntax and wasi-libc patch checks pass in + `2026-07-08T04-34-02-0700-item8-node-check-wasm-runner-final.log` and + `2026-07-08T04-34-02-0700-item8-wasi-libc-patch-check-final.log`. +- **rev:** `zuosnzmq` — `fix(wget): build real GNU Wget for WASI` ### 9. codex-cli — DONE - **Resolved:** the `codex`/`codex-exec` package now has an AgentOS-owned wrapper @@ -349,3 +414,92 @@ real e2e tests that prove Linux-parity behavior — not smoke tests. P0 first — several P1/P3 items depend on it: curl (#6) needs sockets/HTTP; sqlite3 file-DB tests (#11) need pwrite (#4). Fix the runtime layer, then the commands that ride on it, then backfill coverage. One jj rev per item throughout. + +--- + +## Candidate software (future additions) + +Constraint: **C or Rust only** (no Go/Haskell/Python). Real upstream tool or an +established project, same rules as above. **Focus: tools agents invoke headless +and programmatically** — no TUI/visual tools, no dev/build toolchains, no +raw-socket tools. Feasibility: 🟢 easy (fs/compute), 🟡 needs a known pattern +(spawn→`wasi-spawn`, PTY, host TCP/DNS bridge). + +**Atuin-validated priority (from 348K real shell commands):** by actual usage the +clear wins are **jj** (18,929 — 3rd overall after sed/rg/grep) and **tmux** +(1,268), then **perl** (563). Modest but real: ssh/rsync (~23 each), psql (20), +dig (10), redis-cli (8), less (4), openssl (3). **Zero usage in this history:** +zstd, xz, gpg, ffmpeg, age, mlr, socat, nc, screen — generically useful but not +agent-triggered here, so lower priority. Big unserved *demand*: +**ps (1,364) + pkill (866) + pgrep (755) ≈ 3K** — see process management below. + +**Cross-source validation (Homebrew + Debian popcon + agent-sandbox base images + +SWE-agent/OpenHands/Terminal-Bench trajectories):** independent sources converge +tightly on the list below. **Every agent-sandbox image (OpenHands, devcontainers, +GH Actions) pre-installs:** curl, wget, git, jq, tmux, gnupg, xz, zip/unzip, rsync, +ssh, less, vim, tree, procps, psmisc, socat, netcat, ripgrep, bzip2, lz4, sqlite3, +patch, file — near-exact overlap with what we ship/plan. Real **agent +trajectories** are dominated by coreutils (ls/rm/find/mkdir/cat/mv/chmod) + python ++ curl/wget + grep/sed + openssl + ps — all shipped or listed. New adds surfaced: +**imagemagick** ⭐ (C, image ops — high popularity), **openssl** (confirmed +high-use), **aria2 / brotli / parallel / sshpass** (base-image staples). Method +signals worth knowing: (1) agents **edit files ~2:1 over running shell commands**, +and the dominant shell idiom is "write a python repro script, run it, `rm` it" — +so a solid coreutils + python + curl/grep/sed/openssl core matters more than tool +breadth; (2) `git` is **rare inside agent turns** (harnesses extract the diff +out-of-band) but stays essential; (3) a **long tail of project-specific CLIs** +(`dvc`, `sqlglot`, `sanic`, …) comes from pip/npm install, not the registry. + +**Requested (add):** ssh 🟡, rsync 🟡, tmux/screen 🟡 (PTY — session persistence), +gpg 🟡, ffmpeg 🟡 (media transcode — heavy but headless), jj 🟢, dig 🟡, +nslookup 🟡, less ⭐🟡 (pager), openssl ⭐🟡 (TLS/certs/keys/hashing). +tail/head/cat are already in coreutils — confirm present. + +**Text / stream:** less ⭐🟡, **perl** ⭐🟡 (ubiquitous `-pe`/`-ne` text munging — +big C runtime but real; 563 uses in history), miller `mlr` 🟢 (CSV/JSON), +xmlstarlet 🟢, pcre2grep 🟢. (jq/yq/sed/awk/grep/head/tail already covered.) + +**Networking (host TCP/DNS bridge only):** openssl ⭐🟡, ssh 🟡, nc/netcat 🟡 +(TCP/UDP), socat 🟡, whois 🟢, dig/nslookup 🟡, redis-cli / psql client 🟡, +aria2 🟡 (C++ downloader), sshpass 🟢 (ssh password helper). + +**VCS:** git (item above), jj 🟢. + +**Crypto:** gpg 🟡, openssl 🟡, age 🟢 (Rust), minisign 🟢. + +**Compression:** xz, zstd, bzip2, lz4, brotli, p7zip (7z) — all ⭐🟢, common + easy. + +**Media / image:** ffmpeg 🟡 (transcode), imagemagick ⭐🟡 (C, image ops — high +popularity; agents do image work). + +**Files / sync:** rsync 🟡, diff/patch 🟢, rename 🟢, fdupes 🟢 (find/fd tracked +above). + +**Session:** tmux/screen 🟡 (PTY). + +**Process management (add — real procps-ng + psmisc, C; ps/pkill/pgrep ≈ 3K uses):** +- **Need the `/proc` prerequisite (below):** ps, pgrep, pkill, pidof, pstree, + killall, uptime, free, vmstat, w, pwdx, pmap 🟡. +- **Signal-only — already work via the kernel (no /proc):** kill, killall-by-PID. + (kill, sleep, timeout, env, nohup, nproc, nice/renice are coreutils — confirm + they're shipped via uutils rather than re-adding.) + +**⚙️ Runtime prerequisite — implement `/proc` (process-table-backed):** procps +reads `/proc//{stat,cmdline,status,comm}` and enumerates `/proc//`. The +**kernel already owns the process table** (`crates/kernel/src/process_table.rs`), +so expose a read-only procfs view of it to the guest (per-PID stat/cmdline/status ++ directory enumeration). Scope it minimal — just the fields procps parses, backed +by the existing process table, not a full Linux procfs. Unlocks the whole +ps/pkill/pgrep family (and top/htop later if ever wanted). **This is a runtime/VFS +item, do it before the procps packages.** + +**Excluded — not worth it / not possible here:** +- **TUI / visual-only:** gitui, lazygit, eza, dust, ncdu, bat, delta, broot, k9s, + skim/fzf — a terminal UI has no agent value. +- **top / htop — excluded (TUIs).** (ps/pkill/pgrep and the rest of procps-ng + + psmisc are ADD items above, gated on the `/proc` runtime prerequisite.) +- **Raw sockets:** ping, traceroute, mtr, nmap (need raw/ICMP, not just TCP). +- **ptrace:** strace, ltrace, gdb, lldb, valgrind — genuinely impossible on WASI. +- **Dev / build toolchains:** make, cmake, clang/gcc, binutils, pkg-config, + ctags — out of scope. +- **Go-only:** rclone, gh, kubectl — no C/Rust equivalent. diff --git a/packages/core/package.json b/packages/core/package.json index 947d57454f..9b71bf2af0 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -98,6 +98,7 @@ "@agentos-software/tar": "workspace:*", "@agentos-software/tree": "workspace:*", "@agentos-software/vim": "workspace:*", + "@agentos-software/wget": "workspace:*", "@agentos-software/yq": "workspace:*", "@anthropic-ai/claude-agent-sdk": "^0.2.87", "@anthropic-ai/claude-code": "^2.1.86", diff --git a/packages/shell/package.json b/packages/shell/package.json index 4c9e6765b5..254004e6b4 100644 --- a/packages/shell/package.json +++ b/packages/shell/package.json @@ -35,6 +35,7 @@ "@agentos-software/tree": "workspace:*", "@agentos-software/unzip": "workspace:*", "@agentos-software/vim": "workspace:*", + "@agentos-software/wget": "workspace:*", "@agentos-software/yq": "workspace:*", "@agentos-software/zip": "workspace:*", "@rivet-dev/agentos": "workspace:*", diff --git a/packages/shell/src/main.ts b/packages/shell/src/main.ts index f438278d19..f3e6f6996b 100644 --- a/packages/shell/src/main.ts +++ b/packages/shell/src/main.ts @@ -44,6 +44,7 @@ import vim from "@agentos-software/vim"; import tar from "@agentos-software/tar"; import tree from "@agentos-software/tree"; import unzip from "@agentos-software/unzip"; +import wget from "@agentos-software/wget"; import yq from "@agentos-software/yq"; import zip from "@agentos-software/zip"; import type { MountConfig, SoftwareInput } from "@rivet-dev/agentos-core"; @@ -154,6 +155,7 @@ const software: SoftwareInput[] = [ git, httpGet, sqlite3, + wget, ] .map(withLocalCommandFallback) .filter((input): input is SoftwareInput => input !== null); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index acea4b1ad2..6ab0cccb90 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2565,6 +2565,9 @@ importers: '@agentos-software/vim': specifier: workspace:* version: link:../../software/vim + '@agentos-software/wget': + specifier: workspace:* + version: link:../../software/wget '@agentos-software/yq': specifier: workspace:* version: link:../../software/yq @@ -2877,6 +2880,9 @@ importers: '@agentos-software/vim': specifier: workspace:* version: link:../../software/vim + '@agentos-software/wget': + specifier: workspace:* + version: link:../../software/wget '@agentos-software/yq': specifier: workspace:* version: link:../../software/yq @@ -3293,6 +3299,9 @@ importers: '@agentos-software/unzip': specifier: workspace:* version: link:../unzip + '@agentos-software/wget': + specifier: workspace:* + version: link:../wget '@agentos-software/yq': specifier: workspace:* version: link:../yq @@ -3735,6 +3744,27 @@ importers: specifier: ^2.1.9 version: 2.1.9(@types/node@22.19.15) + software/wget: + devDependencies: + '@agentos-software/manifest': + specifier: workspace:* + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness + '@rivet-dev/agentos-toolchain': + specifier: workspace:* + version: link:../../packages/agentos-toolchain + '@types/node': + specifier: ^22.10.2 + version: 22.19.15 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) + software/yq: devDependencies: '@agentos-software/manifest': diff --git a/software/README.md b/software/README.md index de31afdf9c..d104021d1e 100644 --- a/software/README.md +++ b/software/README.md @@ -68,7 +68,7 @@ per-binary entry point; it dispatches to whichever toolchain owns the command: | kind | commands | what it runs | |---|---|---| | Rust | any `software//native/crates/cmd-` command crate (sh, ls, rg, …) | `cargo build -p cmd-` (build-std) + `wasm-opt` | -| C | `zip unzip envsubst sqlite3 curl duckdb` | `make -C c sysroot build/` + per-command install | +| C | `zip unzip envsubst sqlite3 curl wget duckdb` | `make -C c sysroot build/` + per-command install | | codex | `codex`, `codex-exec` | the codex fork build (needs the fork checkout) | | C | `vim` (pinned upstream clone + bridge in `c/vim/`) | `make -C c sysroot build/vim` + install | @@ -90,7 +90,7 @@ packages are pnpm workspace members, so tests and examples resolve them via Exceptions: - `software/codex/wasm/` is the install target of the codex fork's build (`make -C toolchain codex`); `software/codex-cli` stages from it. -- C-built commands (sqlite3, zip, unzip, curl, duckdb) need the patched +- C-built commands (sqlite3, zip, unzip, curl, wget, duckdb) need the patched sysroot; `just toolchain-cmd ` builds it on demand. Without it those packages stay empty placeholders. - `vim` builds from source: `just toolchain-cmd vim` clones the pinned diff --git a/software/everything/package.json b/software/everything/package.json index c491f55094..bc2725d7a8 100644 --- a/software/everything/package.json +++ b/software/everything/package.json @@ -32,6 +32,7 @@ "@agentos-software/tar": "workspace:*", "@agentos-software/gzip": "workspace:*", "@agentos-software/curl": "workspace:*", + "@agentos-software/wget": "workspace:*", "@agentos-software/zip": "workspace:*", "@agentos-software/unzip": "workspace:*", "@agentos-software/jq": "workspace:*", diff --git a/software/everything/src/index.ts b/software/everything/src/index.ts index 58565c3337..e5ea924586 100644 --- a/software/everything/src/index.ts +++ b/software/everything/src/index.ts @@ -7,6 +7,7 @@ import diffutils from "@agentos-software/diffutils"; import tar from "@agentos-software/tar"; import gzip from "@agentos-software/gzip"; import curl from "@agentos-software/curl"; +import wget from "@agentos-software/wget"; import zip from "@agentos-software/zip"; import unzip from "@agentos-software/unzip"; import jq from "@agentos-software/jq"; @@ -27,6 +28,7 @@ const everything = [ tar, gzip, curl, + wget, zip, unzip, jq, @@ -49,6 +51,7 @@ export { tar, gzip, curl, + wget, zip, unzip, jq, diff --git a/software/wget/agentos-package.json b/software/wget/agentos-package.json new file mode 100644 index 0000000000..e8b8178957 --- /dev/null +++ b/software/wget/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "wget" + ], + "registry": { + "title": "wget", + "description": "GNU Wget file downloader.", + "priority": 55, + "image": "/images/registry/wget.svg", + "category": "networking" + } +} diff --git a/software/wget/package.json b/software/wget/package.json new file mode 100644 index 0000000000..8147e9e893 --- /dev/null +++ b/software/wget/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/wget", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU Wget HTTP client for AgentOS VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@agentos/test-harness": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "vitest": "^2.1.9" + } +} diff --git a/software/wget/src/index.ts b/software/wget/src/index.ts new file mode 100644 index 0000000000..dba100e2e1 --- /dev/null +++ b/software/wget/src/index.ts @@ -0,0 +1,5 @@ +import type { SoftwarePackageRef } from "@agentos-software/manifest"; + +const packagePath = new URL("./package.aospkg", import.meta.url).pathname; + +export default { packagePath } satisfies SoftwarePackageRef; diff --git a/software/wget/test/wget.test.ts b/software/wget/test/wget.test.ts new file mode 100644 index 0000000000..c5c13ff20b --- /dev/null +++ b/software/wget/test/wget.test.ts @@ -0,0 +1,162 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { existsSync } from "node:fs"; +import { + createServer, + type IncomingMessage, + type Server, + type ServerResponse, +} from "node:http"; +import { resolve } from "node:path"; +import { createWasmVmRuntime } from "@agentos/test-harness"; +import { + allowAll, + C_BUILD_DIR, + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; + +const WGET_COMMAND_DIRS = [C_BUILD_DIR, COMMANDS_DIR].filter((dir) => + existsSync(dir), +); +const hasWgetBinary = WGET_COMMAND_DIRS.some((dir) => + existsSync(resolve(dir, "wget")), +); +const WGET_EXEC_TIMEOUT_MS = 10_000; + +describeIf(hasWgetBinary, "wget command", () => { + let kernel: Kernel; + let server: Server; + let port: number; + + beforeAll(async () => { + server = createServer((req: IncomingMessage, res: ServerResponse) => { + const url = req.url ?? "/"; + + if (url === "/file.txt") { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("downloaded content"); + return; + } + + if (url === "/data.json") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ status: "ok" })); + return; + } + + if (url === "/redirect") { + res.writeHead(302, { + Location: `http://127.0.0.1:${port}/redirected`, + }); + res.end(); + return; + } + + if (url === "/redirected") { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("arrived after redirect"); + return; + } + + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found"); + }); + + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + port = (server.address() as import("node:net").AddressInfo).port; + }); + + afterAll(async () => { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + }); + + afterEach(async () => { + await kernel?.dispose(); + }); + + async function mountKernel() { + const filesystem = createInMemoryFileSystem(); + kernel = createKernel({ + filesystem, + permissions: allowAll, + loopbackExemptPorts: [port], + }); + await kernel.mount(createWasmVmRuntime({ commandDirs: WGET_COMMAND_DIRS })); + return filesystem; + } + + it("downloads a file using the URL basename", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec(`wget http://127.0.0.1:${port}/file.txt`, { + timeout: WGET_EXEC_TIMEOUT_MS, + }); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/workspace/file.txt")).toBe( + "downloaded content", + ); + }, 15_000); + + it("-O saves to the requested output path", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -O /output.txt http://127.0.0.1:${port}/data.json`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/output.txt")).toContain( + '"status":"ok"', + ); + }, 15_000); + + it("-q suppresses progress output", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -q -O /quiet.txt http://127.0.0.1:${port}/file.txt`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stderr).toBe(""); + expect(await filesystem.readTextFile("/quiet.txt")).toBe( + "downloaded content", + ); + }, 15_000); + + it("reports failure for a 404 URL", async () => { + await mountKernel(); + + const result = await kernel.exec( + `wget http://127.0.0.1:${port}/missing.txt`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/404|not found|error/i); + }, 15_000); + + it("follows redirects by default", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -O /redirected.txt http://127.0.0.1:${port}/redirect`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/redirected.txt")).toBe( + "arrived after redirect", + ); + }, 15_000); +}); diff --git a/software/wget/tsconfig.json b/software/wget/tsconfig.json new file mode 100644 index 0000000000..db523bef56 --- /dev/null +++ b/software/wget/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/toolchain/Makefile b/toolchain/Makefile index 0ff7dcbce9..1761fd6dd2 100644 --- a/toolchain/Makefile +++ b/toolchain/Makefile @@ -213,7 +213,7 @@ wasm: vendor patch-vendor patch-std wasm-opt-check # codex the codex fork build (codex, codex-exec) # `just toolchain-cmd ` calls this. `make wasm` builds the fast # Rust set; `make -C c programs install` builds/installs the fast C set. -C_COMMANDS := zip unzip envsubst sqlite3 curl duckdb vim http_get +C_COMMANDS := zip unzip envsubst sqlite3 curl wget duckdb vim http_get .PHONY: cmd/% cmd/%: diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile index 27c949cd0a..3eefbb8518 100644 --- a/toolchain/c/Makefile +++ b/toolchain/c/Makefile @@ -68,10 +68,10 @@ COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands # Fast real commands installed by the default software gate. Slow/heavy commands # (duckdb, vim) remain available through explicit parent `make cmd/`. -COMMANDS := zip unzip envsubst sqlite3 curl http_get +COMMANDS := zip unzip envsubst sqlite3 curl wget http_get # Programs requiring patched sysroot (Tier 2+ custom host imports) -PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler http_get dns_lookup sqlite3_cli curl fs_probe +PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler http_get dns_lookup sqlite3_cli curl wget fs_probe # Discover all package command and test-program C source files. ALL_SOURCES := $(foreach dir,$(C_SOURCE_DIRS),$(wildcard $(dir)/*.c)) @@ -89,7 +89,7 @@ endif ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) CUSTOM_WASM_PROG_NAMES := else - CUSTOM_WASM_PROG_NAMES := curl + CUSTOM_WASM_PROG_NAMES := curl wget endif WASM_PROG_NAMES := $(sort $(basename $(notdir $(SOURCES))) $(CUSTOM_WASM_PROG_NAMES)) @@ -151,6 +151,11 @@ CURL_RELEASE_URL := https://github.com/curl/curl/releases/download/curl-$(CURL_R CURL_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/curl-upstream CURL_UPSTREAM_OVERLAY_DIR := ../../software/curl/native/c/overlay CURL_UPSTREAM_OVERLAY_FILES := $(wildcard $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.h $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.h) +WGET_VERSION := 1.24.5 +WGET_URL := https://ftp.gnu.org/gnu/wget/wget-$(WGET_VERSION).tar.gz +WGET_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/wget-upstream +WGET_UPSTREAM_OVERLAY_INCLUDE_DIR := overlays/wget/include +WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/*.h) MINIZIP_URL := https://github.com/madler/zlib/archive/refs/tags/v1.3.1.zip # duckdb-wasm currently documents DuckDB v1.5.0 in its README. We build the # matching upstream DuckDB tag ourselves with our patched WASI/POSIX sysroot. @@ -560,6 +565,19 @@ $(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ --output "$(abspath $@)" +$(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-wget-upstream.sh \ + --version "$(WGET_VERSION)" \ + --url "$(WGET_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(WGET_UPSTREAM_BUILD_DIR))" \ + --overlay-include-dir "$(abspath $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --output "$(abspath $@)" + # duckdb: upstream DuckDB CLI built from source with our patched WASI/POSIX sysroot $(BUILD_DIR)/duckdb: libs/duckdb/CMakeLists.txt $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang $(WASI_SDK_DIR)/bin/clang++ cmake/FindThreads.cmake scripts/build-duckdb.sh include/fcntl.h include/ifaddrs.h include/net/if.h include/sys/ioctl.h @mkdir -p $(BUILD_DIR) diff --git a/toolchain/c/overlays/wget/include/unistd.h b/toolchain/c/overlays/wget/include/unistd.h new file mode 100644 index 0000000000..7ff50df4d5 --- /dev/null +++ b/toolchain/c/overlays/wget/include/unistd.h @@ -0,0 +1,14 @@ +#ifndef AGENTOS_WGET_UNISTD_OVERLAY_H +#define AGENTOS_WGET_UNISTD_OVERLAY_H + +#include_next + +/* + * GNU Wget's ptimer fallback only needs to know that Linux-style POSIX timer + * IDs are not compile-time portable here. The underlying sysroot still exposes + * WASI clocks for libc++ and other consumers that use the WASI clockid_t ABI. + */ +#undef _POSIX_TIMERS +#define _POSIX_TIMERS 0 + +#endif diff --git a/toolchain/c/scripts/build-wget-upstream.sh b/toolchain/c/scripts/build-wget-upstream.sh new file mode 100644 index 0000000000..312937df63 --- /dev/null +++ b/toolchain/c/scripts/build-wget-upstream.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: build-wget-upstream.sh \ + --version \ + --url \ + --cache-dir \ + --build-dir \ + --overlay-include-dir \ + --cc \ + --ar \ + --ranlib \ + --output +EOF +} + +VERSION="" +URL="" +CACHE_DIR="" +BUILD_DIR="" +OVERLAY_INCLUDE_DIR="" +CC_CMD="" +AR_CMD="" +RANLIB_CMD="" +OUTPUT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + VERSION="$2" + shift 2 + ;; + --url) + URL="$2" + shift 2 + ;; + --cache-dir) + CACHE_DIR="$2" + shift 2 + ;; + --build-dir) + BUILD_DIR="$2" + shift 2 + ;; + --overlay-include-dir) + OVERLAY_INCLUDE_DIR="$2" + shift 2 + ;; + --cc) + CC_CMD="$2" + shift 2 + ;; + --ar) + AR_CMD="$2" + shift 2 + ;; + --ranlib) + RANLIB_CMD="$2" + shift 2 + ;; + --output) + OUTPUT="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$VERSION" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_INCLUDE_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then + usage >&2 + exit 1 +fi + +fetch() { + local url="$1" + local out="$2" + if command -v curl >/dev/null 2>&1; then + curl --retry 3 --retry-all-errors -fSL "$url" -o "$out" + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$out" + else + echo "Neither curl nor wget is available to fetch $url" >&2 + exit 1 + fi +} + +mkdir -p "$CACHE_DIR" +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +TARBALL="$CACHE_DIR/wget-${VERSION}.tar.gz" +if [[ ! -f "$TARBALL" ]]; then + echo "Fetching upstream GNU Wget ${VERSION} release tarball..." + fetch "$URL" "$TARBALL" +fi + +echo "Extracting upstream GNU Wget ${VERSION}..." +tar -xzf "$TARBALL" -C "$BUILD_DIR" + +SRC_DIR="$BUILD_DIR/wget-${VERSION}" +if [[ ! -d "$SRC_DIR" ]]; then + echo "Expected extracted source at $SRC_DIR" >&2 + exit 1 +fi + +pushd "$SRC_DIR" >/dev/null + +echo "Configuring upstream GNU Wget for wasm32-wasip1..." +CC="$CC_CMD" \ +AR="$AR_CMD" \ +RANLIB="$RANLIB_CMD" \ +PKG_CONFIG=false \ +NETTLE_CFLAGS="" \ +NETTLE_LIBS="" \ +PCRE2_CFLAGS="" \ +PCRE2_LIBS="" \ +PCRE_CFLAGS="" \ +PCRE_LIBS="" \ +CPPFLAGS="-I$OVERLAY_INCLUDE_DIR" \ +gl_cv_func_posix_spawn_works=yes \ +gl_cv_func_posix_spawn_secure_exec=yes \ +gl_cv_func_posix_spawnp_secure_exec=yes \ +gl_cv_func_posix_spawn_file_actions_addclose_works=yes \ +gl_cv_func_posix_spawn_file_actions_adddup2_works=yes \ +gl_cv_func_posix_spawn_file_actions_addopen_works=yes \ +gl_cv_func_select_supports0=yes \ +gl_cv_func_select_detects_ebadf=yes \ +gl_cv_func_pselect_detects_ebadf=yes \ +ac_cv_func_clock_getres=no \ +ac_cv_func_clock_gettime=no \ +CFLAGS="-O2 -flto -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS -DFD_SETSIZE=8192" \ +LIBS="-lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks" \ +./configure \ + --host=wasm32-unknown-wasi \ + --disable-shared \ + --disable-nls \ + --disable-iri \ + --disable-digest \ + --disable-ntlm \ + --disable-opie \ + --disable-pcre \ + --disable-pcre2 \ + --without-ssl \ + --without-libpsl \ + --without-zlib \ + --without-libuuid \ + --without-metalink + +echo "Building upstream GNU Wget support library..." +make -C lib + +echo "Building upstream GNU Wget..." +make -C src wget + +BIN="" +for candidate in "src/wget" "src/.libs/wget" "src/wget.wasm"; do + if [[ -f "$candidate" ]]; then + BIN="$candidate" + break + fi +done + +if [[ -z "$BIN" ]]; then + echo "Unable to locate built wget binary in src/" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$OUTPUT")" +if command -v wasm-opt >/dev/null 2>&1; then + echo "Optimizing Wget WASM binary..." + wasm-opt -O3 --strip-debug --all-features "$BIN" -o "$OUTPUT" +else + cp "$BIN" "$OUTPUT" +fi + +popd >/dev/null + +echo "Built upstream GNU Wget at $OUTPUT" diff --git a/toolchain/std-patches/wasi-libc/0017-resource-limits-and-groups.patch b/toolchain/std-patches/wasi-libc/0017-resource-limits-and-groups.patch new file mode 100644 index 0000000000..0ea7eb5e13 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0017-resource-limits-and-groups.patch @@ -0,0 +1,131 @@ +Expose resource limits and supplementary groups in the AgentOS WASI sysroot. + +GNU Wget's configure script and source expect getgroups(2) and getrlimit(2). +WASI has no kernel-backed process limits or supplementary group database, so the +patched sysroot provides deterministic compatibility shims: one effective group +and fixed per-process limit values. + +diff --git a/libc-bottom-half/sources/host_resource_user.c b/libc-bottom-half/sources/host_resource_user.c +new file mode 100644 +index 0000000..8d91a2f +--- /dev/null ++++ b/libc-bottom-half/sources/host_resource_user.c +@@ -0,0 +1,83 @@ ++// Process resource and group compatibility for AgentOS WASI commands. ++ ++#include ++#include ++#include ++#include ++#include ++ ++static int is_known_rlimit_resource(int resource) { ++ return resource >= 0 && resource < RLIMIT_NLIMITS; ++} ++ ++static rlim_t default_limit_for(int resource) { ++ switch (resource) { ++ case RLIMIT_NOFILE: ++ return 1024; ++ case RLIMIT_CPU: ++ case RLIMIT_FSIZE: ++ case RLIMIT_DATA: ++ case RLIMIT_STACK: ++ case RLIMIT_CORE: ++#ifdef RLIMIT_RSS ++ case RLIMIT_RSS: ++#endif ++#ifdef RLIMIT_NPROC ++ case RLIMIT_NPROC: ++#endif ++#ifdef RLIMIT_MEMLOCK ++ case RLIMIT_MEMLOCK: ++#endif ++#ifdef RLIMIT_AS ++ case RLIMIT_AS: ++#endif ++ case RLIMIT_LOCKS: ++ case RLIMIT_SIGPENDING: ++ case RLIMIT_MSGQUEUE: ++ case RLIMIT_NICE: ++ case RLIMIT_RTPRIO: ++ case RLIMIT_RTTIME: ++ return RLIM_INFINITY; ++ default: ++ return 0; ++ } ++} ++ ++int getgroups(int size, gid_t list[]) { ++ if (size < 0) { ++ errno = EINVAL; ++ return -1; ++ } ++ ++ if (size == 0) { ++ return 1; ++ } ++ ++ if (list == NULL) { ++ errno = EFAULT; ++ return -1; ++ } ++ ++ list[0] = getgid(); ++ return 1; ++} ++ ++int getrlimit(int resource, struct rlimit *rlim) { ++ if (rlim == NULL || !is_known_rlimit_resource(resource)) { ++ errno = EINVAL; ++ return -1; ++ } ++ ++ rlim->rlim_cur = default_limit_for(resource); ++ rlim->rlim_max = default_limit_for(resource); ++ return 0; ++} ++ ++int setrlimit(int resource, const struct rlimit *rlim) { ++ if (rlim == NULL || !is_known_rlimit_resource(resource)) { ++ errno = EINVAL; ++ return -1; ++ } ++ ++ return 0; ++} +diff --git a/libc-top-half/musl/include/sys/resource.h b/libc-top-half/musl/include/sys/resource.h +index d1fb724..deddb7f 100644 +--- a/libc-top-half/musl/include/sys/resource.h ++++ b/libc-top-half/musl/include/sys/resource.h +@@ -1,4 +1,4 @@ +-#ifndef _WASI_EMULATED_PROCESS_CLOCKS ++#if !defined(_WASI_EMULATED_PROCESS_CLOCKS) && defined(__wasilibc_unmodified_upstream) + #error WASI lacks process-associated clocks; to enable emulation of the `getrusage` function using \ + the wall clock, which isn't sensitive to whether the program is running or suspended, \ + compile with -D_WASI_EMULATED_PROCESS_CLOCKS and link with -lwasi-emulated-process-clocks +@@ -21,7 +21,7 @@ extern "C" { + + #include + #include + +-#ifdef __wasilibc_unmodified_upstream /* Use alternate WASI libc headers */ ++#if 1 /* AgentOS exposes POSIX resource-limit declarations in patched WASI. */ + typedef unsigned long long rlim_t; + + struct rlimit { +diff --git a/libc-top-half/musl/include/unistd.h b/libc-top-half/musl/include/unistd.h +index cb62f56..4d858a6 100644 +--- a/libc-top-half/musl/include/unistd.h ++++ b/libc-top-half/musl/include/unistd.h +@@ -173,8 +173,8 @@ uid_t getuid(void); + uid_t geteuid(void); + gid_t getgid(void); + gid_t getegid(void); ++int getgroups(int, gid_t []); + + #ifdef __wasilibc_unmodified_upstream /* WASI has no setuid etc. */ +-int getgroups(int, gid_t []); + int setuid(uid_t); + int seteuid(uid_t); + int setgid(gid_t); diff --git a/toolchain/std-patches/wasi-libc/0018-posix-spawn-and-terminal-headers.patch b/toolchain/std-patches/wasi-libc/0018-posix-spawn-and-terminal-headers.patch new file mode 100644 index 0000000000..6e0e8a6a48 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0018-posix-spawn-and-terminal-headers.patch @@ -0,0 +1,263 @@ +Expose POSIX spawn and terminal compatibility headers in AgentOS wasi-libc. + +The AgentOS process broker already implements posix_spawn through host_process +imports, but the C sysroot still omitted . Upstream programs such as +GNU Wget include standard POSIX headers directly, so install those headers and +provide small compatibility stubs for terminal/process-group helpers that are +meaningless in a headless VM. + +diff --git a/libc-bottom-half/sources/host_terminal_compat.c b/libc-bottom-half/sources/host_terminal_compat.c +new file mode 100644 +index 0000000..3b70693 +--- /dev/null ++++ b/libc-bottom-half/sources/host_terminal_compat.c +@@ -0,0 +1,63 @@ ++// Terminal/process-group compatibility for headless AgentOS WASI commands. ++ ++#include ++#include ++#include ++#include ++ ++pid_t fork(void) { ++ errno = ENOSYS; ++ return -1; ++} ++ ++pid_t getpgrp(void) { ++ return getpid(); ++} ++ ++pid_t setsid(void) { ++ errno = ENOSYS; ++ return -1; ++} ++ ++pid_t tcgetpgrp(int fd) { ++ (void)fd; ++ errno = ENOTTY; ++ return -1; ++} ++ ++int tcsetpgrp(int fd, pid_t pgrp) { ++ (void)fd; ++ (void)pgrp; ++ errno = ENOTTY; ++ return -1; ++} ++ ++int tcgetattr(int fd, struct termios *termios_p) { ++ (void)fd; ++ if (termios_p != NULL) { ++ memset(termios_p, 0, sizeof(*termios_p)); ++ } ++ errno = ENOTTY; ++ return -1; ++} ++ ++int tcsetattr(int fd, int optional_actions, const struct termios *termios_p) { ++ (void)fd; ++ (void)optional_actions; ++ (void)termios_p; ++ errno = ENOTTY; ++ return -1; ++} ++ ++pid_t tcgetsid(int fd) { ++ (void)fd; ++ errno = ENOTTY; ++ return -1; ++} ++ ++char *getpass(const char *prompt) { ++ (void)prompt; ++ static char empty_password[1]; ++ empty_password[0] = '\0'; ++ return empty_password; ++} +diff --git a/libc-bottom-half/sources/host_spawn_wait.c b/libc-bottom-half/sources/host_spawn_wait.c +index 0414908..7b48221 100644 +--- a/libc-bottom-half/sources/host_spawn_wait.c ++++ b/libc-bottom-half/sources/host_spawn_wait.c +@@ -128,6 +128,11 @@ int posix_spawn_file_actions_addchdir_np(posix_spawn_file_actions_t *restrict fa + return __addfdop(fa, op); + } + ++int posix_spawn_file_actions_addchdir(posix_spawn_file_actions_t *restrict fa, ++ const char *restrict path) { ++ return posix_spawn_file_actions_addchdir_np(fa, path); ++} ++ + int posix_spawn_file_actions_addfchdir_np(posix_spawn_file_actions_t *fa, int fd) { + if (fd < 0) return EBADF; + struct __fdop *op = malloc(sizeof(*op)); +diff --git a/libc-top-half/musl/include/spawn.h b/libc-top-half/musl/include/spawn.h +index 225a0f0..55a6fe9 100644 +--- a/libc-top-half/musl/include/spawn.h ++++ b/libc-top-half/musl/include/spawn.h +@@ -65,6 +65,7 @@ int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t *__restrict, int + int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *, int); + int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *, int, int); + ++int posix_spawn_file_actions_addchdir(posix_spawn_file_actions_t *__restrict, const char *__restrict); + #if defined(_BSD_SOURCE) || defined(_GNU_SOURCE) + int posix_spawn_file_actions_addchdir_np(posix_spawn_file_actions_t *__restrict, const char *__restrict); + int posix_spawn_file_actions_addfchdir_np(posix_spawn_file_actions_t *, int); +diff --git a/libc-top-half/musl/include/sys/wait.h b/libc-top-half/musl/include/sys/wait.h +index 27e20b7..5f79380 100644 +--- a/libc-top-half/musl/include/sys/wait.h ++++ b/libc-top-half/musl/include/sys/wait.h +@@ -21,11 +21,12 @@ pid_t waitpid (pid_t, int *, int ); + +-#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) \ +- || defined(_XOPEN_SOURCE) || defined(_GNU_SOURCE) \ +- || defined(_BSD_SOURCE) ++#if defined(__wasilibc_unmodified_upstream) \ ++ && (defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) \ ++ || defined(_XOPEN_SOURCE) || defined(_GNU_SOURCE) \ ++ || defined(_BSD_SOURCE)) + #include + int waitid (idtype_t, id_t, siginfo_t *, int); + #endif + +-#if defined(_GNU_SOURCE) || defined(_BSD_SOURCE) ++#if (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) && defined(__wasilibc_unmodified_upstream) + #include + pid_t wait3 (int *, int, struct rusage *); + pid_t wait4 (pid_t, int *, int, struct rusage *); +diff --git a/libc-top-half/musl/include/fcntl.h b/libc-top-half/musl/include/fcntl.h +index eac8775..a7ad6d5 100644 +--- a/libc-top-half/musl/include/fcntl.h ++++ b/libc-top-half/musl/include/fcntl.h +@@ -31,6 +31,39 @@ int open(const char *, int, ...); + int openat(int, const char *, int, ...); + int posix_fadvise(int, off_t, off_t, int); + int posix_fallocate(int, off_t, off_t); ++ ++#ifndef F_DUPFD ++#define F_DUPFD 0 ++#endif ++ ++#ifndef O_BINARY ++#define O_BINARY 0 ++#endif ++#ifndef O_TEXT ++#define O_TEXT 0 ++#endif ++ ++#ifndef F_RDLCK ++#define F_RDLCK 0 ++#endif ++#ifndef F_WRLCK ++#define F_WRLCK 1 ++#endif ++#ifndef F_UNLCK ++#define F_UNLCK 2 ++#endif ++ ++#ifndef F_GETLK ++#define F_GETLK 12 ++#define F_SETLK 13 ++#define F_SETLKW 14 ++#endif ++ ++#ifndef F_GETLK64 ++#define F_GETLK64 F_GETLK ++#define F_SETLK64 F_SETLK ++#define F_SETLKW64 F_SETLKW ++#endif + + #ifdef __wasilibc_unmodified_upstream /* Use alternate WASI libc headers */ + #define O_SEARCH O_PATH +diff --git a/libc-top-half/musl/include/setjmp.h b/libc-top-half/musl/include/setjmp.h +index bbeffc9..f938367 100644 +--- a/libc-top-half/musl/include/setjmp.h ++++ b/libc-top-half/musl/include/setjmp.h +@@ -7,12 +7,6 @@ extern "C" { + + #include + +-#ifndef __wasilibc_unmodified_upstream +-/* WASI has no setjmp */ +-#if !defined(__wasm_exception_handling__) +-#error Setjmp/longjmp support requires Exception handling support, which is [not yet standardized](https://github.com/WebAssembly/proposals?tab=readme-ov-file#phase-3---implementation-phase-cg--wg). To enable it, compile with `-mllvm -wasm-enable-sjlj` and use an engine that implements the Exception handling proposal. +-#endif +-#endif + #include + + typedef struct __jmp_buf_tag { +diff --git a/libc-top-half/musl/include/unistd.h b/libc-top-half/musl/include/unistd.h +index ded1723..742e3f0 100644 +--- a/libc-top-half/musl/include/unistd.h ++++ b/libc-top-half/musl/include/unistd.h +@@ -137,7 +137,7 @@ int pause(void); + #endif + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no fork/exec */ + pid_t fork(void); ++#ifdef __wasilibc_unmodified_upstream /* WASI has no fork/exec */ + pid_t _Fork(void); + int execve(const char *, char *const [], char *const []); + int execv(const char *, char *const []); +@@ -151,10 +151,10 @@ _Noreturn void _exit(int); + pid_t getpid(void); + pid_t getppid(void); + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no process groups */ + pid_t getpgrp(void); ++pid_t setsid(void); ++#ifdef __wasilibc_unmodified_upstream /* WASI has no full process groups */ + pid_t getpgid(pid_t); + int setpgid(pid_t, pid_t); +-pid_t setsid(void); + pid_t getsid(pid_t); + #endif +@@ -165,11 +165,9 @@ pid_t getsid(pid_t); + char *ttyname(int); + int ttyname_r(int, char *, size_t); + #endif + int isatty(int); +-#ifdef __wasilibc_unmodified_upstream /* WASI has no process groups */ + pid_t tcgetpgrp(int); + int tcsetpgrp(int, pid_t); +-#endif + + uid_t getuid(void); + uid_t geteuid(void); +@@ -240,12 +239,12 @@ int chroot(const char *); + #endif + int getpagesize(void); ++char *getpass(const char *); + #ifdef __wasilibc_unmodified_upstream /* WASI has no processes */ + int getdtablesize(void); + int sethostname(const char *, size_t); + int getdomainname(char *, size_t); + int setdomainname(const char *, size_t); + int setgroups(size_t, const gid_t *); +-char *getpass(const char *); + int daemon(int, int); + void setusershell(void); + void endusershell(void); +diff --git a/scripts/install-include-headers.sh b/scripts/install-include-headers.sh +index 30b2da5..4a9f079 100755 +--- a/scripts/install-include-headers.sh ++++ b/scripts/install-include-headers.sh +@@ -56,15 +56,15 @@ MUSL_OMIT_HEADERS+=("sys/procfs.h" "sys/user.h" "sys/kd.h" "sys/vt.h" \ + "sys/soundcard.h" "sys/sem.h" "sys/shm.h" "sys/msg.h" "sys/ipc.h" \ + "sys/ptrace.h" "sys/statfs.h" "bits/kd.h" "bits/vt.h" "bits/soundcard.h" \ + "bits/sem.h" "bits/shm.h" "bits/msg.h" "bits/ipc.h" "bits/ptrace.h" \ + "bits/statfs.h" "sys/vfs.h" "syslog.h" "sys/syslog.h" "wait.h" \ +- "sys/wait.h" "ucontext.h" "sys/ucontext.h" "paths.h" "utmp.h" "utmpx.h" \ ++ "ucontext.h" "sys/ucontext.h" "paths.h" "utmp.h" "utmpx.h" \ + "lastlog.h" "sys/acct.h" "sys/cachectl.h" "sys/epoll.h" "sys/reboot.h" \ + "sys/swap.h" "sys/sendfile.h" "sys/inotify.h" "sys/quota.h" "sys/klog.h" \ + "sys/fsuid.h" "sys/io.h" "sys/prctl.h" "sys/mtio.h" "sys/mount.h" \ + "sys/fanotify.h" "sys/personality.h" "elf.h" "link.h" "bits/link.h" \ + "scsi/scsi.h" "scsi/scsi_ioctl.h" "scsi/sg.h" "sys/auxv.h" \ + "shadow.h" "grp.h" "mntent.h" "resolv.h" "pty.h" "ulimit.h" "sys/xattr.h" \ +- "wordexp.h" "spawn.h" "sys/membarrier.h" "sys/signalfd.h" "termios.h" \ +- "sys/termios.h" "bits/termios.h" "net/if.h" "net/if_arp.h" \ ++ "wordexp.h" "sys/membarrier.h" "sys/signalfd.h" \ ++ "net/if.h" "net/if_arp.h" \ + "net/ethernet.h" "net/route.h" "netinet/if_ether.h" "netinet/ether.h" \ + "sys/timerfd.h" "libintl.h" "sys/sysmacros.h" "aio.h") diff --git a/toolchain/std-patches/wasi-libc/0019-standard-signal-count.patch b/toolchain/std-patches/wasi-libc/0019-standard-signal-count.patch new file mode 100644 index 0000000000..9bda573fd7 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0019-standard-signal-count.patch @@ -0,0 +1,19 @@ +Advertise the standard signal range in the AgentOS WASI sysroot. + +The patched WASI signal surface exposes conventional non-realtime signal names +through SIGSYS (31) and keeps realtime signal APIs hidden. Report _NSIG as 32 so +portable gnulib-generated headers do not reject the sysroot as a realtime-signal +platform. + +diff --git a/libc-top-half/musl/arch/wasm32/bits/signal.h b/libc-top-half/musl/arch/wasm32/bits/signal.h +index 478e1c8..7501d99 100644 +--- a/libc-top-half/musl/arch/wasm32/bits/signal.h ++++ b/libc-top-half/musl/arch/wasm32/bits/signal.h +@@ -35,6 +35,6 @@ + #define SIGSYS 31 + #define SIGUNUSED SIGSYS + +-#define _NSIG 65 ++#define _NSIG 32 + + #endif diff --git a/toolchain/std-patches/wasi-libc/0020-process-signal-mask.patch b/toolchain/std-patches/wasi-libc/0020-process-signal-mask.patch new file mode 100644 index 0000000000..79dc7e3435 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0020-process-signal-mask.patch @@ -0,0 +1,97 @@ +Expose process signal-mask helpers in the AgentOS WASI sysroot. + +The patched signal surface uses musl's struct sigset_t layout. Let upstream +programs use the sysroot's sigprocmask/pthread_sigmask declarations instead of +falling back to gnulib replacements that assume sigset_t is an integer. + +diff --git a/libc-bottom-half/sources/host_sigprocmask.c b/libc-bottom-half/sources/host_sigprocmask.c +new file mode 100644 +index 0000000..7e5a037 +--- /dev/null ++++ b/libc-bottom-half/sources/host_sigprocmask.c +@@ -0,0 +1,61 @@ ++// Process signal mask compatibility for AgentOS WASI commands. ++ ++#include ++#include ++ ++static sigset_t process_signal_mask; ++ ++static int copy_sigset(sigset_t *dst, const sigset_t *src) { ++ if (dst == NULL || src == NULL) { ++ errno = EINVAL; ++ return -1; ++ } ++ *dst = *src; ++ return 0; ++} ++ ++int sigprocmask(int how, const sigset_t *restrict set, sigset_t *restrict oldset) { ++ sigset_t next_mask; ++ ++ if (oldset != NULL) { ++ *oldset = process_signal_mask; ++ } ++ ++ if (set == NULL) { ++ return 0; ++ } ++ ++ next_mask = process_signal_mask; ++ switch (how) { ++ case SIG_BLOCK: ++ for (int signum = 1; signum < NSIG; signum++) { ++ if (sigismember(set, signum) == 1) { ++ sigaddset(&next_mask, signum); ++ } ++ } ++ break; ++ case SIG_UNBLOCK: ++ for (int signum = 1; signum < NSIG; signum++) { ++ if (sigismember(set, signum) == 1) { ++ sigdelset(&next_mask, signum); ++ } ++ } ++ break; ++ case SIG_SETMASK: ++ if (copy_sigset(&next_mask, set) != 0) return -1; ++ break; ++ default: ++ errno = EINVAL; ++ return -1; ++ } ++ ++ process_signal_mask = next_mask; ++ return 0; ++} ++ ++int pthread_sigmask(int how, const sigset_t *restrict set, sigset_t *restrict oldset) { ++ if (sigprocmask(how, set, oldset) != 0) { ++ return errno; ++ } ++ return 0; ++} +diff --git a/libc-top-half/musl/include/signal.h b/libc-top-half/musl/include/signal.h +index 6830b5b..7f1bfcc 100644 +--- a/libc-top-half/musl/include/signal.h ++++ b/libc-top-half/musl/include/signal.h +@@ -263,9 +263,9 @@ int sigaddset(sigset_t *, int); + int sigdelset(sigset_t *, int); + int sigismember(const sigset_t *, int); + int sigaction(int, const struct sigaction *__restrict, struct sigaction *__restrict); ++int sigprocmask(int, const sigset_t *__restrict, sigset_t *__restrict); + + #ifdef __wasilibc_unmodified_upstream /* WASI has no signal sets */ +-int sigprocmask(int, const sigset_t *__restrict, sigset_t *__restrict); + int sigsuspend(const sigset_t *); + int sigpending(sigset_t *); + int sigwait(const sigset_t *__restrict, int *__restrict); +@@ -273,7 +273,7 @@ int sigtimedwait(const sigset_t *__restrict, siginfo_t *__restrict, const struct + int sigqueue(pid_t, int, union sigval); + #endif + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no threads yet */ + int pthread_sigmask(int, const sigset_t *__restrict, sigset_t *__restrict); ++#ifdef __wasilibc_unmodified_upstream /* WASI has no full thread signal APIs */ + int pthread_kill(pthread_t, int); + #endif diff --git a/toolchain/std-patches/wasi-libc/0021-utc-timezone-compat.patch b/toolchain/std-patches/wasi-libc/0021-utc-timezone-compat.patch new file mode 100644 index 0000000000..486457f9c9 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0021-utc-timezone-compat.patch @@ -0,0 +1,56 @@ +Expose UTC timezone compatibility in the AgentOS WASI sysroot. + +AgentOS currently uses a minimal UTC timezone implementation. Provide the +standard tzset/tzname/daylight/timezone surface as no-op UTC compatibility so +upstream time code can compile without shipping timezone tables. + +diff --git a/libc-bottom-half/sources/host_timezone_compat.c b/libc-bottom-half/sources/host_timezone_compat.c +new file mode 100644 +index 0000000..0b4ef74 +--- /dev/null ++++ b/libc-bottom-half/sources/host_timezone_compat.c +@@ -0,0 +1,9 @@ ++// UTC timezone compatibility for AgentOS WASI commands. ++ ++long timezone = 0; ++int daylight = 0; ++char *tzname[2] = { "UTC", "UTC" }; ++ ++void tzset(void) { ++ // AgentOS VMs currently expose UTC only. ++} +diff --git a/libc-top-half/musl/include/time.h b/libc-top-half/musl/include/time.h +index f534148..919ae5d 100644 +--- a/libc-top-half/musl/include/time.h ++++ b/libc-top-half/musl/include/time.h +@@ -97,9 +97,7 @@ struct tm *localtime_r (const time_t *__restrict, struct tm *__restrict); + char *asctime_r (const struct tm *__restrict, char *__restrict); + char *ctime_r (const time_t *, char *); + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no timezone tables */ + void tzset (void); +-#endif + + struct itimerspec { + struct timespec it_interval; +@@ -143,19 +141,15 @@ int timer_gettime (timer_t, struct itimerspec *); + int timer_getoverrun (timer_t); + #endif + +-#ifdef __wasilibc_unmodified_upstream /* WASI has no timezone tables */ + extern char *tzname[2]; +-#endif + + #endif + + + #if defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) + char *strptime (const char *__restrict, const char *__restrict, struct tm *__restrict); +-#ifdef __wasilibc_unmodified_upstream /* WASI has no timezone tables */ + extern int daylight; + extern long timezone; +-#endif + extern int getdate_err; + struct tm *getdate (const char *); + #endif + diff --git a/toolchain/std-patches/wasi-libc/0022-overrideable-fd-set-size.patch b/toolchain/std-patches/wasi-libc/0022-overrideable-fd-set-size.patch new file mode 100644 index 0000000000..f40fb79627 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0022-overrideable-fd-set-size.patch @@ -0,0 +1,34 @@ +Allow AgentOS builds to raise FD_SETSIZE when needed. + +AgentOS guest file descriptors are virtual ids and may be greater than the +traditional 1024 select limit. The sysroot fd_set representation is list-based, +so consumers that can tolerate a larger fd_set should be able to opt in by +predefining FD_SETSIZE. + +diff --git a/libc-bottom-half/headers/public/__macro_FD_SETSIZE.h b/libc-bottom-half/headers/public/__macro_FD_SETSIZE.h +index c5854c0..bc93827 100644 +--- a/libc-bottom-half/headers/public/__macro_FD_SETSIZE.h ++++ b/libc-bottom-half/headers/public/__macro_FD_SETSIZE.h +@@ -1,6 +1,8 @@ + #ifndef __wasilibc___macro_FD_SETSIZE_h + #define __wasilibc___macro_FD_SETSIZE_h + ++#ifndef FD_SETSIZE + #define FD_SETSIZE 1024 ++#endif + + #endif +diff --git a/libc-top-half/musl/include/sys/select.h b/libc-top-half/musl/include/sys/select.h +index 11ebe93..351c547 100644 +--- a/libc-top-half/musl/include/sys/select.h ++++ b/libc-top-half/musl/include/sys/select.h +@@ -15,7 +15,9 @@ extern "C" { + + #include + ++#ifndef FD_SETSIZE + #define FD_SETSIZE 1024 ++#endif + + #ifdef __wasilibc_unmodified_upstream /* Use alternate WASI libc headers */ + typedef unsigned long fd_mask; diff --git a/toolchain/std-patches/wasi-libc/0023-host-net-read-write-sockets.patch b/toolchain/std-patches/wasi-libc/0023-host-net-read-write-sockets.patch new file mode 100644 index 0000000000..4ac076f9f9 --- /dev/null +++ b/toolchain/std-patches/wasi-libc/0023-host-net-read-write-sockets.patch @@ -0,0 +1,82 @@ +Route POSIX read/write through host_net for socket descriptors. + +Upstream C programs commonly use read() and write() on socket fds. AgentOS +host_net sockets are brokered outside the regular WASI fd syscalls, so try +host_net first and fall back to regular descriptors when the fd is not a +host_net socket. + +diff --git a/libc-bottom-half/cloudlibc/src/libc/unistd/read.c b/libc-bottom-half/cloudlibc/src/libc/unistd/read.c +index 826e84b..367dcaf 100644 +--- a/libc-bottom-half/cloudlibc/src/libc/unistd/read.c ++++ b/libc-bottom-half/cloudlibc/src/libc/unistd/read.c +@@ -3,16 +3,32 @@ + // SPDX-License-Identifier: BSD-2-Clause + + #include + #include ++#include + #include + ++__attribute__((__import_module__("host_net"), __import_name__("net_recv"))) ++uint32_t __host_net_read_recv(uint32_t fd, uint8_t *buf_ptr, uint32_t buf_len, ++ uint32_t flags, uint32_t *ret_received); ++ + ssize_t read(int fildes, void *buf, size_t nbyte) { ++ if (nbyte > UINT32_MAX) { ++ errno = EINVAL; ++ return -1; ++ } ++ ++ uint32_t socket_bytes_read; ++ uint32_t socket_error = __host_net_read_recv((uint32_t)fildes, (uint8_t *)buf, ++ (uint32_t)nbyte, 0, &socket_bytes_read); ++ if (socket_error == 0) return (ssize_t)socket_bytes_read; ++ if (socket_error != EBADF) { errno = (int)socket_error; return -1; } ++ + __wasi_iovec_t iov = {.buf = buf, .buf_len = nbyte}; + size_t bytes_read; + __wasi_errno_t error = __wasi_fd_read(fildes, &iov, 1, &bytes_read); + if (error != 0) { + errno = error == ENOTCAPABLE ? EBADF : error; + return -1; + } + return bytes_read; + } +diff --git a/libc-bottom-half/cloudlibc/src/libc/unistd/write.c b/libc-bottom-half/cloudlibc/src/libc/unistd/write.c +index 63c4bcd..604b4d5 100644 +--- a/libc-bottom-half/cloudlibc/src/libc/unistd/write.c ++++ b/libc-bottom-half/cloudlibc/src/libc/unistd/write.c +@@ -3,17 +3,33 @@ + // SPDX-License-Identifier: BSD-2-Clause + + #include + #include ++#include + #include + ++__attribute__((__import_module__("host_net"), __import_name__("net_send"))) ++uint32_t __host_net_write_send(uint32_t fd, const uint8_t *buf_ptr, uint32_t buf_len, ++ uint32_t flags, uint32_t *ret_sent); ++ + ssize_t write(int fildes, const void *buf, size_t nbyte) { ++ if (nbyte > UINT32_MAX) { ++ errno = EINVAL; ++ return -1; ++ } ++ ++ uint32_t socket_bytes_written; ++ uint32_t socket_error = __host_net_write_send((uint32_t)fildes, (const uint8_t *)buf, ++ (uint32_t)nbyte, 0, &socket_bytes_written); ++ if (socket_error == 0) return (ssize_t)socket_bytes_written; ++ if (socket_error != EBADF) { errno = (int)socket_error; return -1; } ++ + __wasi_ciovec_t iov = {.buf = buf, .buf_len = nbyte}; + size_t bytes_written; + __wasi_errno_t error = + __wasi_fd_write(fildes, &iov, 1, &bytes_written); + if (error != 0) { + errno = error == ENOTCAPABLE ? EBADF : error; + return -1; + } + return bytes_written; + }