diff --git a/.github/workflows/fat.yml b/.github/workflows/fat.yml new file mode 100644 index 000000000..0d5a8b4ad --- /dev/null +++ b/.github/workflows/fat.yml @@ -0,0 +1,83 @@ +name: "Fat archive" + +# CWIST_FAT=1 folds the vendored static archives into libcwist.a and +# `make fat-dist` stages it with the headers the public API pulls in, so an +# embedded target needs only the bundle plus the app source at link time; +# CWIST_APP_FAT=1 links the example apps against that fat archive alone. +# This job builds the bundle, proves a server app links and serves with +# nothing but the bundle copied next to its source, and builds an example +# with CWIST_APP_FAT=1. + +on: + push: + branches: [main, master, dev] + pull_request: + branches: [main, master, dev] + workflow_dispatch: + +permissions: + contents: read + +jobs: + fat: + name: Fat archive build + embedded-style link smoke + if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake libcurl4-openssl-dev libnghttp2-dev libbrotli-dev libzstd-dev + + - name: Build the fat archive and embedded bundle + run: make fat-dist -j"$(nproc)" + + - name: Verify vendored archives are folded in + run: | + # Members of every vendored archive must be present, not just the + # CWIST objects: the archive alone has to satisfy an app link. + ar t libcwist.a | grep -q '^cJSON\.o$' + ar t libcwist.a | grep -q '^bio_ssl\.cc\.o$' + ar t libcwist.a | grep -q '^lsqpack\.c\.o$' + ar t libcwist.a | grep -q '^UriCommon\.c\.o$' + nm --defined-only libcwist.a | grep -qw 'T SSL_new' + nm --defined-only libcwist.a | grep -qw 'T lsquic_engine_new' + nm --defined-only libcwist.a | grep -qw 'T natsConnection_Connect' + nm --defined-only libcwist.a | grep -qw 'T sqlite3_open' + + - name: Embedded-style link smoke (bundle + app source only) + run: | + # Mirror the embedded workflow: copy just the fat-dist bundle and + # the app source onto the target, link against system libs only. + work=$(mktemp -d) + cp -r dist/cwist-fat "$work/bundle" + cp example/micro/01-hello/main.c "$work/" + cd "$work" + "${CC:-gcc}" -O2 -std=gnu17 -Ibundle/include -o hello main.c \ + bundle/lib/libcwist.a \ + $(pkg-config --libs libcurl libnghttp2 libbrotlienc libbrotlicommon libbrotlidec libzstd) \ + -pthread -ldl -lm -lstdc++ -lz + ./hello & + pid=$! + sleep 2 + test "$(curl -s -m 3 http://127.0.0.1:8080/)" = "Hello, CWIST!" + kill "$pid" + + - name: CWIST_APP_FAT example build + run: | + # The app link must reference libcwist.a and no vendored archive. + out=$(make CWIST_APP_FAT=1 example/micro/01-hello/hello 2>&1 | tail -1) + echo "$out" + echo "$out" | grep -q 'libcwist\.a' + ! echo "$out" | grep -q 'lib/[a-z0-9_/-]*/[^ ]*\.a' + ./example/micro/01-hello/hello & + pid=$! + sleep 2 + test "$(curl -s -m 3 http://127.0.0.1:8080/)" = "Hello, CWIST!" + kill "$pid" diff --git a/Makefile b/Makefile index 0ae79ca10..a4ff9b443 100644 --- a/Makefile +++ b/Makefile @@ -331,6 +331,7 @@ SRCS = src/core/sstring/sstring.c \ EMCC ?= emcc EMAR ?= emar NODE ?= node +RANLIB ?= ranlib WASM_BUILD_DIR = .wasm-build WASM_SRCS = src/core/sstring/sstring.c \ src/core/seq/seq.c \ @@ -638,6 +639,11 @@ component-stream-smoke: $(COMPONENT_BUILD_DIR)/stream_guest.component.wasm --alias:c4punks:cwist/host=$(abspath tests/wasm_component_stream_host.js) \ --external:@bytecodealliance/jco-node-fs* \ --outfile=$(STREAM_JCO_DIR)/bundle.mjs + # preview3-shim pulls in preview2-shim's io worker, which spawns + # new Worker(new URL("./worker-thread.bundle.js", import.meta.url)); + # esbuild does not emit that file, so place it next to the bundle. + cp $(STREAM_JCO_DIR)/node_modules/@bytecodealliance/preview2-shim/dist/io/worker-thread.bundle.js \ + $(STREAM_JCO_DIR)/worker-thread.bundle.js BUNDLE=$(abspath $(STREAM_JCO_DIR)/bundle.mjs) \ $(NODE) --experimental-wasm-jspi tests/wasm_component_stream_test.js @@ -681,6 +687,21 @@ component-browser-test: component-browser-smoke # Object Files and Target OBJS = $(SRCS:.c=.o) LIB_NAME = libcwist.a + +# Rebuild every object when the compile flags change. CWIST_WEBTRANSPORT adds +# and removes struct members in http3.c, so flipping it without a clean left a +# library holding objects built both ways: the layouts disagreed and the result +# was a stack smash at runtime, not a compile error. .build-flags records the +# flags and every object depends on it, so a changed flag forces a rebuild. +.build-flags: FORCE + @printf '%s' '$(CFLAGS)' | cmp -s - $@ || printf '%s' '$(CFLAGS)' > $@ +# Records the fat-archive toggle so flipping CWIST_FAT re-archives libcwist.a +# without forcing a recompile of unchanged objects. +.build-fat: FORCE + @printf '%s' 'fat=$(CWIST_FAT)' | cmp -s - $@ || printf '%s' 'fat=$(CWIST_FAT)' > $@ +FORCE: +.PHONY: FORCE fat-dist +$(OBJS): .build-flags LIBTTAK_DIR = lib/libttak LIBTTAK_LIB = $(LIBTTAK_DIR)/lib/libttak.a LIBTTAK_EXTRA_CFLAGS = @@ -738,6 +759,19 @@ ifeq ($(CWIST_WEBRTC),1) LIBS += $(USRSCTP_LIB) endif +# --- Fat (embedded-friendly) build options ----------------------------------- +# CWIST_FAT=1 folds every vendored static archive into libcwist.a, so copying +# the single archive plus the CWIST headers onto an embedded target is enough +# to link a server app; only system libraries stay external. +# CWIST_APP_FAT=1 builds the example apps against the fat archive alone (the +# vendored archives drop off their link line); it implies CWIST_FAT=1. +ifeq ($(CWIST_APP_FAT),1) +CWIST_FAT = 1 +APP_LIBS = $(filter-out $(EXTERNAL_LIBS),$(LIBS)) +else +APP_LIBS = $(LIBS) +endif + # --- Build Targets --- all: $(LIBTTAK_LIB) $(CJSON_LIB) $(URIPARSER_LIB) $(SQLITE_DIR)/sqlite3.c $(LSQUIC_LIB) $(LIB_NAME) @@ -789,10 +823,39 @@ lib/lsquic/include/lsquic.h: git submodule update --init --recursive $(LSQUIC_DIR); \ fi -$(LIB_NAME): $(EXTERNAL_LIBS) $(OBJS) - @echo "Creating thin static library..." +$(LIB_NAME): .build-fat $(EXTERNAL_LIBS) $(OBJS) @rm -f $@ - ar rcs $@ $(OBJS) + @if [ "$(CWIST_FAT)" = "1" ]; then \ + echo "Creating fat static library (CWIST objects + vendored archives)..."; \ + if [ "$(UNAME_S)" = "Darwin" ]; then \ + libtool -static -o $@ $(sort $(OBJS)) $(EXTERNAL_LIBS); \ + else \ + { echo "CREATE $@"; \ + for o in $(sort $(OBJS)); do echo "ADDMOD $$o"; done; \ + for l in $(EXTERNAL_LIBS); do echo "ADDLIB $$l"; done; \ + echo "SAVE"; echo "END"; } | $(AR) -M && $(RANLIB) $@; \ + fi; \ + else \ + echo "Creating static library..."; \ + $(AR) rcs $@ $(OBJS); \ + fi + +# The embedded bundle: the fat archive plus every header the public API +# pulls in (the public headers reference cjson, openssl, sqlite3 and ttak), +# laid out as the compiler expects. Copy FAT_DIST_DIR onto the target board +# and build apps with -I/include /lib/libcwist.a; only system +# libraries stay external. +FAT_DIST_DIR ?= dist/cwist-fat +fat-dist: CWIST_FAT = 1 +fat-dist: $(LIB_NAME) + rm -rf $(FAT_DIST_DIR) + mkdir -p $(FAT_DIST_DIR)/lib $(FAT_DIST_DIR)/include/cjson + cp $(LIB_NAME) $(FAT_DIST_DIR)/lib/ + cp -R include/cwist $(FAT_DIST_DIR)/include/ + cp $(CJSON_DIR)/cJSON.h $(FAT_DIST_DIR)/include/cjson/ + cp -R $(LIBTTAK_DIR)/include/ttak $(FAT_DIST_DIR)/include/ + cp -R $(BORINGSSL_DIR)/include/openssl $(FAT_DIST_DIR)/include/ + cp $(SQLITE_DIR)/sqlite3.h $(FAT_DIST_DIR)/include/ $(LIBTTAK_LIB): @echo "Building libttak..." @@ -1543,37 +1606,37 @@ EXAMPLE_BINS = example/simple-server/simple-server \ examples: $(EXAMPLE_BINS) example/simple-server/simple-server: $(LIB_NAME) example/simple-server/main.c - $(CC) $(CFLAGS) -o $@ example/simple-server/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/simple-server/main.c $(LIB_NAME) $(APP_LIBS) example/http/step-1-hello-world/hello-world: $(LIB_NAME) example/http/step-1-hello-world/main.c - $(CC) $(CFLAGS) -o $@ example/http/step-1-hello-world/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/http/step-1-hello-world/main.c $(LIB_NAME) $(APP_LIBS) example/http/step-2-mux-router/mux-router: $(LIB_NAME) example/http/step-2-mux-router/main.c - $(CC) $(CFLAGS) -o $@ example/http/step-2-mux-router/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/http/step-2-mux-router/main.c $(LIB_NAME) $(APP_LIBS) example/http/step-3-query-params/query-params: $(LIB_NAME) example/http/step-3-query-params/main.c - $(CC) $(CFLAGS) -o $@ example/http/step-3-query-params/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/http/step-3-query-params/main.c $(LIB_NAME) $(APP_LIBS) example/http/step-4-json-api/json-api: $(LIB_NAME) example/http/step-4-json-api/main.c - $(CC) $(CFLAGS) -o $@ example/http/step-4-json-api/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/http/step-4-json-api/main.c $(LIB_NAME) $(APP_LIBS) example/jwt/step-2-http-auth/jwt-auth: $(LIB_NAME) example/jwt/step-2-http-auth/main.c - $(CC) $(CFLAGS) -o $@ example/jwt/step-2-http-auth/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/jwt/step-2-http-auth/main.c $(LIB_NAME) $(APP_LIBS) example/cde-json-viewer/cde-json-viewer: $(LIB_NAME) example/cde-json-viewer/main.c - $(CC) $(CFLAGS) -o $@ example/cde-json-viewer/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/cde-json-viewer/main.c $(LIB_NAME) $(APP_LIBS) example/db/step-1-open-query/open-query: $(LIB_NAME) example/db/step-1-open-query/main.c - $(CC) $(CFLAGS) -o $@ example/db/step-1-open-query/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/db/step-1-open-query/main.c $(LIB_NAME) $(APP_LIBS) example/db/step-2-migrations/migrations: $(LIB_NAME) example/db/step-2-migrations/main.c - $(CC) $(CFLAGS) -o $@ example/db/step-2-migrations/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/db/step-2-migrations/main.c $(LIB_NAME) $(APP_LIBS) example/db/step-4-json-insert/json-insert: $(LIB_NAME) example/db/step-4-json-insert/main.c - $(CC) $(CFLAGS) -o $@ example/db/step-4-json-insert/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/db/step-4-json-insert/main.c $(LIB_NAME) $(APP_LIBS) example/rps-showcase/rps-showcase: $(LIB_NAME) example/rps-showcase/main.c - $(CC) $(CFLAGS) -o $@ example/rps-showcase/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/rps-showcase/main.c $(LIB_NAME) $(APP_LIBS) # Micro examples MICRO_BINS = example/micro/01-hello/hello \ @@ -1621,55 +1684,55 @@ examples-check: $(LIB_NAME) $(EXAMPLE_BINS) $(MICRO_BINS) done example/micro/01-hello/hello: $(LIB_NAME) example/micro/01-hello/main.c - $(CC) $(CFLAGS) -o $@ example/micro/01-hello/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/01-hello/main.c $(LIB_NAME) $(APP_LIBS) example/micro/02-routes/routes: $(LIB_NAME) example/micro/02-routes/main.c - $(CC) $(CFLAGS) -o $@ example/micro/02-routes/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/02-routes/main.c $(LIB_NAME) $(APP_LIBS) example/micro/03-path-params/path-params: $(LIB_NAME) example/micro/03-path-params/main.c - $(CC) $(CFLAGS) -o $@ example/micro/03-path-params/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/03-path-params/main.c $(LIB_NAME) $(APP_LIBS) example/micro/04-query-params/query-params: $(LIB_NAME) example/micro/04-query-params/main.c - $(CC) $(CFLAGS) -o $@ example/micro/04-query-params/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/04-query-params/main.c $(LIB_NAME) $(APP_LIBS) example/micro/05-json/json: $(LIB_NAME) example/micro/05-json/main.c - $(CC) $(CFLAGS) -o $@ example/micro/05-json/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/05-json/main.c $(LIB_NAME) $(APP_LIBS) example/micro/06-orm-insert/orm-insert: $(LIB_NAME) example/micro/06-orm-insert/main.c - $(CC) $(CFLAGS) -o $@ example/micro/06-orm-insert/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/06-orm-insert/main.c $(LIB_NAME) $(APP_LIBS) example/micro/07-orm-query/orm-query: $(LIB_NAME) example/micro/07-orm-query/main.c - $(CC) $(CFLAGS) -o $@ example/micro/07-orm-query/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/07-orm-query/main.c $(LIB_NAME) $(APP_LIBS) example/micro/08-orm-update-delete/orm-update-delete: $(LIB_NAME) example/micro/08-orm-update-delete/main.c - $(CC) $(CFLAGS) -o $@ example/micro/08-orm-update-delete/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/08-orm-update-delete/main.c $(LIB_NAME) $(APP_LIBS) example/micro/09-html-builder/html-builder: $(LIB_NAME) example/micro/09-html-builder/main.c - $(CC) $(CFLAGS) -o $@ example/micro/09-html-builder/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/09-html-builder/main.c $(LIB_NAME) $(APP_LIBS) example/micro/10-static-files/static-files: $(LIB_NAME) example/micro/10-static-files/main.c - $(CC) $(CFLAGS) -o $@ example/micro/10-static-files/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/10-static-files/main.c $(LIB_NAME) $(APP_LIBS) example/micro/11-jwt-auth/jwt-auth: $(LIB_NAME) example/micro/11-jwt-auth/main.c - $(CC) $(CFLAGS) -o $@ example/micro/11-jwt-auth/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/11-jwt-auth/main.c $(LIB_NAME) $(APP_LIBS) example/micro/12-middleware/middleware: $(LIB_NAME) example/micro/12-middleware/main.c - $(CC) $(CFLAGS) -o $@ example/micro/12-middleware/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/12-middleware/main.c $(LIB_NAME) $(APP_LIBS) example/micro/13-nuke-db/nuke-db: $(LIB_NAME) example/micro/13-nuke-db/main.c - $(CC) $(CFLAGS) -o $@ example/micro/13-nuke-db/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/13-nuke-db/main.c $(LIB_NAME) $(APP_LIBS) example/micro/14-websocket/websocket: $(LIB_NAME) example/micro/14-websocket/main.c - $(CC) $(CFLAGS) -o $@ example/micro/14-websocket/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/14-websocket/main.c $(LIB_NAME) $(APP_LIBS) example/micro/15-pqc-tls/pqc-tls: $(LIB_NAME) example/micro/15-pqc-tls/main.c - $(CC) $(CFLAGS) -o $@ example/micro/15-pqc-tls/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/15-pqc-tls/main.c $(LIB_NAME) $(APP_LIBS) example/micro/16-rdbms-auto/rdbms-auto: $(LIB_NAME) example/micro/16-rdbms-auto/main.c - $(CC) $(CFLAGS) -o $@ example/micro/16-rdbms-auto/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/16-rdbms-auto/main.c $(LIB_NAME) $(APP_LIBS) example/micro/17-blog-crud/blog-crud: $(LIB_NAME) example/micro/17-blog-crud/main.c - $(CC) $(CFLAGS) -o $@ example/micro/17-blog-crud/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/17-blog-crud/main.c $(LIB_NAME) $(APP_LIBS) clean-examples: rm -f $(EXAMPLE_BINS) $(MICRO_BINS) diff --git a/docs/api/wasm-component.md b/docs/api/wasm-component.md index 8393aca64..b39fbb83d 100644 --- a/docs/api/wasm-component.md +++ b/docs/api/wasm-component.md @@ -86,6 +86,14 @@ waitable-set/callback bindings that compile for wasm32-wasip3 with wasi-sdk 34; jco surfaces custom host imports as plain ESM imports, which is how the smoke (and any bundler consumer) wires `sendChunk`. +The npm adapter exposes this world as `handle.stream(init, onChunk)`: +`createStreamHost()` from `cwist-wasm/component` is the `c4punks:cwist/host` +import, and binding the same object as `streamHost` routes each chunk to +the callback of the stream in flight. A failing callback reaches the guest +as the `err` arm of `send-chunk`, which aborts the pump, and `stream()` +rejects with the callback's own error. `make component-stream-smoke-p3` +drives the guest through the adapter, including that abort path. + Only wasm32-wasip3 builds this world: 0.2 components cannot lower async imports. Bindings are generated per world (`wit-bindgen --world`), and `wasm-tools component embed` picks the world explicitly @@ -141,7 +149,9 @@ overhead is what crosses the default limit. Emscripten stays supported regardless. 3. **0.3 cutover (conditional).** Both preconditions held up under evaluation and the streaming boundary has landed in - `cwist-guest-stream`. Remaining before the cutover: track JSPI + `cwist-guest-stream`, with a JS entry point in the npm adapter + (`handle.stream()`, the counterpart of `Module.cwistStreamChunk` on + the Emscripten path). Remaining before the cutover: track JSPI shipping unflagged in node and browsers, and port the SSE examples off the EM_JS callback. Only then does the Emscripten build leave CI. diff --git a/tests/wasm_component_stream_host.js b/tests/wasm_component_stream_host.js index 80d735026..27edff4f7 100644 --- a/tests/wasm_component_stream_host.js +++ b/tests/wasm_component_stream_host.js @@ -3,13 +3,12 @@ * 'c4punks:cwist/host' to this file when bundling the transpiled stream * guest, which is how a bundler consumer wires a custom host import. * - * Chunks are collected on globalThis so the test driver can read them - * through its own (esbuild-bundled) copy of this module. */ + * The import is the cwist-wasm adapter's createStreamHost(); the host + * object is published on globalThis so the test driver can bind its + * handle to the same instance the bundled guest calls into. */ +import { createStreamHost } from '../wasm/npm/component.js'; -export const chunks = []; -globalThis.__cwistStreamChunks = chunks; +const host = createStreamHost(); +globalThis.__cwistStreamHost = host; -export async function sendChunk(chunk) { - /* chunk arrives as a Uint8Array copy owned by the canonical ABI. */ - chunks.push(Buffer.from(chunk)); -} +export const sendChunk = host.sendChunk; diff --git a/tests/wasm_component_stream_test.js b/tests/wasm_component_stream_test.js index cdcfe3926..734c579fa 100644 --- a/tests/wasm_component_stream_test.js +++ b/tests/wasm_component_stream_test.js @@ -1,8 +1,8 @@ /* Drives the jco-transpiled cwist-guest-stream component (bundled by * `make component-stream-smoke` with tests/wasm_component_stream_host.js - * aliased in as the c4punks:cwist/host import) and asserts that a - * streaming dispatch delivers the serialized response head-first through - * the async host send-chunk. + * aliased in as the c4punks:cwist/host import) through the cwist-wasm + * component adapter, and asserts that handle.stream() delivers the + * serialized response head-first through the async host send-chunk. * * Run under node's JSPI: node --experimental-wasm-jspi * tests/wasm_component_stream_test.js with BUNDLE pointing at the esbuild @@ -10,7 +10,7 @@ 'use strict'; const path = require('path'); -const { buildRequestBytes } = require(path.join(__dirname, '..', 'wasm', 'npm', 'index.js')); +const { createCwistFromComponent } = require(path.join(__dirname, '..', 'wasm', 'npm', 'component.js')); const BUNDLE = process.env.BUNDLE; if (!BUNDLE) { @@ -24,21 +24,28 @@ if (!BUNDLE) { /* Command component: run() executes main() (route registration). */ await component.run.run(); + const handle = createCwistFromComponent({ + dispatch: component.guest.dispatch, + useSession: component.guest.useSession, + dispatchStream: component.guestStream.dispatchStream, + streamHost: globalThis.__cwistStreamHost, + }); + /* Sync WIT exports of a 0.3 component lower to async functions under * jco, same as the plain guest. */ - const resBytes = await component.guest.dispatch( - buildRequestBytes({ method: 'GET', path: '/hello' })); - const head = new TextDecoder().decode(resBytes); - if (!head.startsWith('HTTP/1.1 200')) { - throw new Error('plain dispatch broken in stream world: ' + head.slice(0, 40)); + const res = await handle({ method: 'GET', path: '/hello' }); + if (res.status !== 200) { + throw new Error('plain dispatch broken in stream world: ' + res.status); } /* Streaming dispatch: the whole serialized response arrives through - * host.send-chunk, head first, body last. */ - const chunks = globalThis.__cwistStreamChunks; - chunks.length = 0; - await component.guestStream.dispatchStream( - buildRequestBytes({ method: 'GET', path: '/events' })); + * host.send-chunk, head first, body last. The callback awaits, so the + * guest's pump waits on it (backpressure). */ + const chunks = []; + await handle.stream({ method: 'GET', path: '/events' }, async (chunk) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + chunks.push(Buffer.from(chunk)); + }); if (chunks.length < 2) { throw new Error('expected head + body chunks, got ' + chunks.length); @@ -54,6 +61,30 @@ if (!BUNDLE) { throw new Error('missing SSE body: ' + JSON.stringify(wire.slice(-80))); } + /* A failing callback aborts the dispatch, and stream() rejects with the + * callback's own error rather than the guest's dispatch-error. */ + const boom = new Error('client went away'); + let seen = 0; + let rejected = null; + try { + await handle.stream({ method: 'GET', path: '/events' }, () => { + seen++; + throw boom; + }); + } catch (e) { + rejected = e; + } + if (rejected !== boom) { + throw new Error('expected the callback error, got: ' + rejected); + } + if (seen !== 1) { + throw new Error('dispatch kept pumping after the callback failed: ' + seen + ' chunks'); + } + + /* The instance stays usable after an aborted stream. */ + const after = await handle({ method: 'GET', path: '/hello' }); + if (after.status !== 200) throw new Error('dispatch after aborted stream: ' + after.status); + console.log('component-stream-smoke: PASS (streamed /events through host.send-chunk, ' + chunks.length + ' chunks)'); process.exit(0); diff --git a/wasm/npm/README.md b/wasm/npm/README.md index 76383ed21..c3bd380ff 100644 --- a/wasm/npm/README.md +++ b/wasm/npm/README.md @@ -76,11 +76,41 @@ for browsers as-is (verified with esbuild in the CWIST repo, returns a Promise and `await` is required; 0.2 guests stay synchronous. See `docs/api/wasm-component.md` in the CWIST repo for building the guest. +### Streaming (0.3 guests) + +A guest built for the `cwist-guest-stream` world (wasm32-wasip3) streams a +response through its `c4punks:cwist/host` import instead of returning it. +`createStreamHost()` returns that import. Wire it in when you instantiate or +bundle the component, and give the same object to the adapter: + +```js +const { createCwistFromComponent, createStreamHost } = require('cwist-wasm/component'); + +const host = createStreamHost(); // { sendChunk }: the c4punks:cwist/host import +// ...instantiate or bundle the transpiled guest with `host` as that import... +const handle = createCwistFromComponent({ + dispatch: guest.dispatch, + useSession: guest.useSession, + dispatchStream: guestStream.dispatchStream, + streamHost: host, +}); + +await handle.stream({ method: 'GET', path: '/events' }, async (chunk) => { + // chunk is a Uint8Array: the serialized head first, then body slices + await writer.write(chunk); +}); +``` + +The guest waits for each callback, so a callback that returns a Promise +applies backpressure. If the callback throws or rejects, the dispatch stops +and `stream()` rejects with that error. Streaming dispatches on one handle +run one at a time. + ## Scope This wrapper is intentionally thin: routing, middleware, validation, and rendering all stay in your C handlers. It only serializes requests into HTTP/1.1 bytes, calls `cwist_app_dispatch_memory()` inside WASM, and parses -the serialized response back into JS objects. Streaming and session -persistence are not handled here; see `docs/api/wasm.md` in the CWIST repo -for the current caveats. +the serialized response back into JS objects. Session persistence is not +handled here, and streaming is only available on the component path above; +see `docs/api/wasm.md` in the CWIST repo for the current caveats. diff --git a/wasm/npm/component.d.ts b/wasm/npm/component.d.ts index 5077effd4..ca73bb005 100644 --- a/wasm/npm/component.d.ts +++ b/wasm/npm/component.d.ts @@ -15,23 +15,50 @@ export interface CwistResponse { body: Uint8Array; } +/** + * Receives one serialized chunk of a streaming response: the head (status + * line + headers) first, then body slices. Return a Promise to apply + * backpressure; a throw or rejection aborts the dispatch. + */ +export type CwistChunkCallback = (chunk: Uint8Array) => void | Promise; + export interface CwistHandle { - (init?: CwistRequestInit): CwistResponse; + /** + * Synchronous against a 0.2 (wasip2) guest; returns a Promise against a + * 0.3 (wasip3) guest, whose exports jco lowers to async functions. + */ + (init?: CwistRequestInit): CwistResponse | Promise; /** * Pin the session signing secret (host-injected contract; issue #93). * Pass null to let CWIST generate a per-instance random secret (dev * mode). Must be called before the first session-bearing dispatch. */ useSession(secret: string | null): void; + /** + * Streaming dispatch through the cwist-guest-stream world (0.3 only): + * resolves once the last chunk has been delivered to onChunk, rejects + * with the dispatch-error or with the callback's own error. Needs + * dispatchStream and streamHost on the bound component. + */ + stream(init: CwistRequestInit | undefined, onChunk: CwistChunkCallback): Promise; +} + +/** The c4punks:cwist/host import of the cwist-guest-stream world. */ +export interface CwistStreamHost { + sendChunk(chunk: Uint8Array): Promise; } /** Minimal shape of the guest interface of a jco-transpiled cwist-guest * component (docs/api/wasm-component.md, issue #203). */ export interface CwistComponent { - dispatch(request: Uint8Array): Uint8Array; - useSession(secret: string | null): void; + dispatch(request: Uint8Array): Uint8Array | Promise; + useSession(secret: string | null): void | Promise; /** Optional declarative session secret, applied once at binding time. */ cwistSessionSecret?: string; + /** guest-stream's dispatch-stream; only cwist-guest-stream guests have it. */ + dispatchStream?(request: Uint8Array): Promise; + /** The createStreamHost() object wired as the component's host import. */ + streamHost?: CwistStreamHost; } /** @@ -40,3 +67,9 @@ export interface CwistComponent { * dispatch-error variant. */ export function createCwistFromComponent(component: CwistComponent): CwistHandle; + +/** + * Create the c4punks:cwist/host import for a cwist-guest-stream component; + * pass the same object as `streamHost` when binding the component. + */ +export function createStreamHost(): CwistStreamHost; diff --git a/wasm/npm/component.js b/wasm/npm/component.js index 105d443fd..064aa4319 100644 --- a/wasm/npm/component.js +++ b/wasm/npm/component.js @@ -11,11 +11,78 @@ * const res = handle({ method: 'GET', path: '/hello' }); * // res.status, res.headers, res.body (Uint8Array) — same as createCwist() * // Against a 0.3 (wasip3) component handle() returns a Promise: await it. + * + * Streaming (cwist-guest-stream world, 0.3 only): the guest pushes chunks + * through the c4punks:cwist/host import, which is wired when the component + * is instantiated or bundled. createStreamHost() returns that import; + * handing the same host to the adapter routes each chunk to the callback + * of the handle.stream() call in flight: + * + * const host = createStreamHost(); // { sendChunk } -> host import + * const handle = createCwistFromComponent({ ...guest, + * dispatchStream: guestStream.dispatchStream, streamHost: host }); + * await handle.stream({ path: '/events' }, (chunk) => { ... }); */ 'use strict'; const { buildRequestBytes, parseResponseBytes } = require('./index.js'); +/* Internal: the dispatch runner behind each stream host. Kept off the host + * object so the host is exactly the shape of the WIT import. + * + * The registry must live on globalThis: a bundler (esbuild alias, webpack) + * inlines a second copy of this module into the guest bundle when the host + * import references createStreamHost(), while the consumer's adapter comes + * from the npm package — two module instances, two WeakMaps, and the lookup + * in stream() would miss every registration made through the bundled copy. */ +const STREAM_RUNNERS = (globalThis.__cwistStreamRunners ||= new WeakMap()); + +/* jco turns a thrown value carrying `payload` into the err arm of the + * import's result<_, string>; the guest then aborts the dispatch. */ +function streamFailure(message) { + return { payload: message }; +} + +/** + * Create the c4punks:cwist/host import for a cwist-guest-stream component. + * Pass the returned object (or its sendChunk) as the host import, and the + * same object as `streamHost` to createCwistFromComponent(). + * @returns {{sendChunk: (chunk: Uint8Array) => Promise}} + */ +function createStreamHost() { + let sink = null; + let sinkError = null; + + /* Awaited by the guest: delaying resolution is the backpressure. */ + async function sendChunk(chunk) { + if (!sink) throw streamFailure('cwist-wasm: send-chunk outside a streaming dispatch'); + try { + await sink(chunk); + } catch (e) { + sinkError = { error: e }; + throw streamFailure('cwist-wasm: chunk callback failed: ' + String(e && e.message ? e.message : e)); + } + } + + const host = { sendChunk }; + STREAM_RUNNERS.set(host, async function run(onChunk, dispatch) { + sink = onChunk; + sinkError = null; + try { + await dispatch(); + } catch (e) { + /* A failed callback aborts the guest's pump, which then reports a + * dispatch-error; surface the callback's own error instead. */ + if (sinkError) throw sinkError.error; + throw e; + } finally { + sink = null; + } + if (sinkError) throw sinkError.error; + }); + return host; +} + /** * Create a CWIST request handler bound to an instantiated cwist-guest * component (jco transpile output of a guest built for wit/cwist.wit). @@ -71,8 +138,33 @@ function createCwistFromComponent(component) { return parseResponseBytes(component.dispatch(request)); } + /* One sink per host, so streaming dispatches run one at a time; they also + * wait for any pending useSession(). */ + let streamQueue = Promise.resolve(); + + function stream(init, onChunk) { + if (typeof component.dispatchStream !== 'function') { + throw new Error( + 'cwist-wasm: component does not expose dispatchStream; build the guest for the cwist-guest-stream world (wasm32-wasip3)' + ); + } + const run = component.streamHost && STREAM_RUNNERS.get(component.streamHost); + if (!run) { + throw new Error( + 'cwist-wasm: streaming needs streamHost, the createStreamHost() object wired as the c4punks:cwist/host import' + ); + } + if (typeof onChunk !== 'function') throw new Error('cwist-wasm: stream() needs a chunk callback'); + const request = buildRequestBytes(init || {}); + const result = Promise.all([chain, streamQueue]) + .then(() => run(onChunk, () => component.dispatchStream(request))); + streamQueue = result.then(() => {}, () => {}); + return result; + } + handle.useSession = useSession; + handle.stream = stream; return handle; } -module.exports = { createCwistFromComponent }; +module.exports = { createCwistFromComponent, createStreamHost };