cherry-pick: wasm streaming dispatch (#426) + CWIST_FAT fat archive (#427) to main

.github/workflows/fat.yml

@@ -0,0 +1,83 @@
+name: "Fat archive"
+
+# CWIST_FAT=1 folds the vendored static archives into libcwist.a and
+# `make fat-dist` stages it with the headers the public API pulls in, so an
+# embedded target needs only the bundle plus the app source at link time;
+# CWIST_APP_FAT=1 links the example apps against that fat archive alone.
+# This job builds the bundle, proves a server app links and serves with
+# nothing but the bundle copied next to its source, and builds an example
+# with CWIST_APP_FAT=1.
+
+on:
+  push:
+    branches: [main, master, dev]
+  pull_request:
+    branches: [main, master, dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  fat:
+    name: Fat archive build + embedded-style link smoke
+    if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]')
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Install build dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y cmake libcurl4-openssl-dev libnghttp2-dev libbrotli-dev libzstd-dev
+
+    - name: Build the fat archive and embedded bundle
+      run: make fat-dist -j"$(nproc)"
+
+    - name: Verify vendored archives are folded in
+      run: |
+        # Members of every vendored archive must be present, not just the
+        # CWIST objects: the archive alone has to satisfy an app link.
+        ar t libcwist.a | grep -q '^cJSON\.o$'
+        ar t libcwist.a | grep -q '^bio_ssl\.cc\.o$'
+        ar t libcwist.a | grep -q '^lsqpack\.c\.o$'
+        ar t libcwist.a | grep -q '^UriCommon\.c\.o$'
+        nm --defined-only libcwist.a | grep -qw 'T SSL_new'
+        nm --defined-only libcwist.a | grep -qw 'T lsquic_engine_new'
+        nm --defined-only libcwist.a | grep -qw 'T natsConnection_Connect'
+        nm --defined-only libcwist.a | grep -qw 'T sqlite3_open'
+
+    - name: Embedded-style link smoke (bundle + app source only)
+      run: |
+        # Mirror the embedded workflow: copy just the fat-dist bundle and
+        # the app source onto the target, link against system libs only.
+        work=$(mktemp -d)
+        cp -r dist/cwist-fat "$work/bundle"
+        cp example/micro/01-hello/main.c "$work/"
+        cd "$work"
+        "${CC:-gcc}" -O2 -std=gnu17 -Ibundle/include -o hello main.c \
+          bundle/lib/libcwist.a \
+          $(pkg-config --libs libcurl libnghttp2 libbrotlienc libbrotlicommon libbrotlidec libzstd) \
+          -pthread -ldl -lm -lstdc++ -lz
+        ./hello &
+        pid=$!
+        sleep 2
+        test "$(curl -s -m 3 http://127.0.0.1:8080/)" = "Hello, CWIST!"
+        kill "$pid"
+
+    - name: CWIST_APP_FAT example build
+      run: |
+        # The app link must reference libcwist.a and no vendored archive.
+        out=$(make CWIST_APP_FAT=1 example/micro/01-hello/hello 2>&1 | tail -1)
+        echo "$out"
+        echo "$out" | grep -q 'libcwist\.a'
+        ! echo "$out" | grep -q 'lib/[a-z0-9_/-]*/[^ ]*\.a'
+        ./example/micro/01-hello/hello &
+        pid=$!
+        sleep 2
+        test "$(curl -s -m 3 http://127.0.0.1:8080/)" = "Hello, CWIST!"
+        kill "$pid"

Makefile

@@ -331,6 +331,7 @@ SRCS = src/core/sstring/sstring.c \
 EMCC ?= emcc
 EMAR ?= emar
 NODE ?= node
+RANLIB ?= ranlib
 WASM_BUILD_DIR = .wasm-build
 WASM_SRCS = src/core/sstring/sstring.c \
        src/core/seq/seq.c \
@@ -638,6 +639,11 @@ component-stream-smoke: $(COMPONENT_BUILD_DIR)/stream_guest.component.wasm
 	    --alias:c4punks:cwist/host=$(abspath tests/wasm_component_stream_host.js) \
 	    --external:@bytecodealliance/jco-node-fs* \
 	    --outfile=$(STREAM_JCO_DIR)/bundle.mjs
+	# preview3-shim pulls in preview2-shim's io worker, which spawns
+	# new Worker(new URL("./worker-thread.bundle.js", import.meta.url));
+	# esbuild does not emit that file, so place it next to the bundle.
+	cp $(STREAM_JCO_DIR)/node_modules/@bytecodealliance/preview2-shim/dist/io/worker-thread.bundle.js \
+	    $(STREAM_JCO_DIR)/worker-thread.bundle.js
 	BUNDLE=$(abspath $(STREAM_JCO_DIR)/bundle.mjs) \
 	    $(NODE) --experimental-wasm-jspi tests/wasm_component_stream_test.js
 
@@ -681,6 +687,21 @@ component-browser-test: component-browser-smoke
 # Object Files and Target
 OBJS = $(SRCS:.c=.o)
 LIB_NAME = libcwist.a
+
+# Rebuild every object when the compile flags change.  CWIST_WEBTRANSPORT adds
+# and removes struct members in http3.c, so flipping it without a clean left a
+# library holding objects built both ways: the layouts disagreed and the result
+# was a stack smash at runtime, not a compile error.  .build-flags records the
+# flags and every object depends on it, so a changed flag forces a rebuild.
+.build-flags: FORCE
+	@printf '%s' '$(CFLAGS)' | cmp -s - $@ || printf '%s' '$(CFLAGS)' > $@
+# Records the fat-archive toggle so flipping CWIST_FAT re-archives libcwist.a
+# without forcing a recompile of unchanged objects.
+.build-fat: FORCE
+	@printf '%s' 'fat=$(CWIST_FAT)' | cmp -s - $@ || printf '%s' 'fat=$(CWIST_FAT)' > $@
+FORCE:
+.PHONY: FORCE fat-dist
+$(OBJS): .build-flags
 LIBTTAK_DIR = lib/libttak
 LIBTTAK_LIB = $(LIBTTAK_DIR)/lib/libttak.a
 LIBTTAK_EXTRA_CFLAGS =
@@ -738,6 +759,19 @@ ifeq ($(CWIST_WEBRTC),1)
     LIBS += $(USRSCTP_LIB)
 endif
 
+# --- Fat (embedded-friendly) build options -----------------------------------
+# CWIST_FAT=1 folds every vendored static archive into libcwist.a, so copying
+# the single archive plus the CWIST headers onto an embedded target is enough
+# to link a server app; only system libraries stay external.
+# CWIST_APP_FAT=1 builds the example apps against the fat archive alone (the
+# vendored archives drop off their link line); it implies CWIST_FAT=1.
+ifeq ($(CWIST_APP_FAT),1)
+CWIST_FAT = 1
+APP_LIBS = $(filter-out $(EXTERNAL_LIBS),$(LIBS))
+else
+APP_LIBS = $(LIBS)
+endif
+
 # --- Build Targets ---
 
 all: $(LIBTTAK_LIB) $(CJSON_LIB) $(URIPARSER_LIB) $(SQLITE_DIR)/sqlite3.c $(LSQUIC_LIB) $(LIB_NAME)
@@ -789,10 +823,39 @@ lib/lsquic/include/lsquic.h:
 		git submodule update --init --recursive $(LSQUIC_DIR); \
 	fi
 
-$(LIB_NAME): $(EXTERNAL_LIBS) $(OBJS)
-	@echo "Creating thin static library..."
+$(LIB_NAME): .build-fat $(EXTERNAL_LIBS) $(OBJS)
 	@rm -f $@
-	ar rcs $@ $(OBJS)
+	@if [ "$(CWIST_FAT)" = "1" ]; then \
+	    echo "Creating fat static library (CWIST objects + vendored archives)..."; \
+	    if [ "$(UNAME_S)" = "Darwin" ]; then \
+	        libtool -static -o $@ $(sort $(OBJS)) $(EXTERNAL_LIBS); \
+	    else \
+	        { echo "CREATE $@"; \
+	          for o in $(sort $(OBJS)); do echo "ADDMOD $$o"; done; \
+	          for l in $(EXTERNAL_LIBS); do echo "ADDLIB $$l"; done; \
+	          echo "SAVE"; echo "END"; } | $(AR) -M && $(RANLIB) $@; \
+	    fi; \
+	else \
+	    echo "Creating static library..."; \
+	    $(AR) rcs $@ $(OBJS); \
+	fi
+
+# The embedded bundle: the fat archive plus every header the public API
+# pulls in (the public headers reference cjson, openssl, sqlite3 and ttak),
+# laid out as the compiler expects. Copy FAT_DIST_DIR onto the target board
+# and build apps with -I<dir>/include <dir>/lib/libcwist.a; only system
+# libraries stay external.
+FAT_DIST_DIR ?= dist/cwist-fat
+fat-dist: CWIST_FAT = 1
+fat-dist: $(LIB_NAME)
+	rm -rf $(FAT_DIST_DIR)
+	mkdir -p $(FAT_DIST_DIR)/lib $(FAT_DIST_DIR)/include/cjson
+	cp $(LIB_NAME) $(FAT_DIST_DIR)/lib/
+	cp -R include/cwist $(FAT_DIST_DIR)/include/
+	cp $(CJSON_DIR)/cJSON.h $(FAT_DIST_DIR)/include/cjson/
+	cp -R $(LIBTTAK_DIR)/include/ttak $(FAT_DIST_DIR)/include/
+	cp -R $(BORINGSSL_DIR)/include/openssl $(FAT_DIST_DIR)/include/
+	cp $(SQLITE_DIR)/sqlite3.h $(FAT_DIST_DIR)/include/
 
 $(LIBTTAK_LIB):
 	@echo "Building libttak..."
@@ -1543,37 +1606,37 @@ EXAMPLE_BINS = example/simple-server/simple-server \
 examples: $(EXAMPLE_BINS)
 
 example/simple-server/simple-server: $(LIB_NAME) example/simple-server/main.c
-	$(CC) $(CFLAGS) -o $@ example/simple-server/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/simple-server/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/http/step-1-hello-world/hello-world: $(LIB_NAME) example/http/step-1-hello-world/main.c
-	$(CC) $(CFLAGS) -o $@ example/http/step-1-hello-world/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/http/step-1-hello-world/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/http/step-2-mux-router/mux-router: $(LIB_NAME) example/http/step-2-mux-router/main.c
-	$(CC) $(CFLAGS) -o $@ example/http/step-2-mux-router/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/http/step-2-mux-router/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/http/step-3-query-params/query-params: $(LIB_NAME) example/http/step-3-query-params/main.c
-	$(CC) $(CFLAGS) -o $@ example/http/step-3-query-params/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/http/step-3-query-params/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/http/step-4-json-api/json-api: $(LIB_NAME) example/http/step-4-json-api/main.c
-	$(CC) $(CFLAGS) -o $@ example/http/step-4-json-api/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/http/step-4-json-api/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/jwt/step-2-http-auth/jwt-auth: $(LIB_NAME) example/jwt/step-2-http-auth/main.c
-	$(CC) $(CFLAGS) -o $@ example/jwt/step-2-http-auth/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/jwt/step-2-http-auth/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/cde-json-viewer/cde-json-viewer: $(LIB_NAME) example/cde-json-viewer/main.c
-	$(CC) $(CFLAGS) -o $@ example/cde-json-viewer/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/cde-json-viewer/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/db/step-1-open-query/open-query: $(LIB_NAME) example/db/step-1-open-query/main.c
-	$(CC) $(CFLAGS) -o $@ example/db/step-1-open-query/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/db/step-1-open-query/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/db/step-2-migrations/migrations: $(LIB_NAME) example/db/step-2-migrations/main.c
-	$(CC) $(CFLAGS) -o $@ example/db/step-2-migrations/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/db/step-2-migrations/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/db/step-4-json-insert/json-insert: $(LIB_NAME) example/db/step-4-json-insert/main.c
-	$(CC) $(CFLAGS) -o $@ example/db/step-4-json-insert/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/db/step-4-json-insert/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/rps-showcase/rps-showcase: $(LIB_NAME) example/rps-showcase/main.c
-	$(CC) $(CFLAGS) -o $@ example/rps-showcase/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/rps-showcase/main.c $(LIB_NAME) $(APP_LIBS)
 
 # Micro examples
 MICRO_BINS = example/micro/01-hello/hello \
@@ -1621,55 +1684,55 @@ examples-check: $(LIB_NAME) $(EXAMPLE_BINS) $(MICRO_BINS)
 	done
 
 example/micro/01-hello/hello: $(LIB_NAME) example/micro/01-hello/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/01-hello/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/01-hello/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/02-routes/routes: $(LIB_NAME) example/micro/02-routes/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/02-routes/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/02-routes/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/03-path-params/path-params: $(LIB_NAME) example/micro/03-path-params/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/03-path-params/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/03-path-params/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/04-query-params/query-params: $(LIB_NAME) example/micro/04-query-params/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/04-query-params/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/04-query-params/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/05-json/json: $(LIB_NAME) example/micro/05-json/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/05-json/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/05-json/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/06-orm-insert/orm-insert: $(LIB_NAME) example/micro/06-orm-insert/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/06-orm-insert/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/06-orm-insert/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/07-orm-query/orm-query: $(LIB_NAME) example/micro/07-orm-query/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/07-orm-query/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/07-orm-query/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/08-orm-update-delete/orm-update-delete: $(LIB_NAME) example/micro/08-orm-update-delete/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/08-orm-update-delete/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/08-orm-update-delete/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/09-html-builder/html-builder: $(LIB_NAME) example/micro/09-html-builder/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/09-html-builder/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/09-html-builder/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/10-static-files/static-files: $(LIB_NAME) example/micro/10-static-files/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/10-static-files/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/10-static-files/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/11-jwt-auth/jwt-auth: $(LIB_NAME) example/micro/11-jwt-auth/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/11-jwt-auth/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/11-jwt-auth/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/12-middleware/middleware: $(LIB_NAME) example/micro/12-middleware/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/12-middleware/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/12-middleware/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/13-nuke-db/nuke-db: $(LIB_NAME) example/micro/13-nuke-db/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/13-nuke-db/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/13-nuke-db/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/14-websocket/websocket: $(LIB_NAME) example/micro/14-websocket/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/14-websocket/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/14-websocket/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/15-pqc-tls/pqc-tls: $(LIB_NAME) example/micro/15-pqc-tls/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/15-pqc-tls/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/15-pqc-tls/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/16-rdbms-auto/rdbms-auto: $(LIB_NAME) example/micro/16-rdbms-auto/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/16-rdbms-auto/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/16-rdbms-auto/main.c $(LIB_NAME) $(APP_LIBS)
 
 example/micro/17-blog-crud/blog-crud: $(LIB_NAME) example/micro/17-blog-crud/main.c
-	$(CC) $(CFLAGS) -o $@ example/micro/17-blog-crud/main.c $(LIB_NAME) $(LIBS)
+	$(CC) $(CFLAGS) -o $@ example/micro/17-blog-crud/main.c $(LIB_NAME) $(APP_LIBS)
 
 clean-examples:
 	rm -f $(EXAMPLE_BINS) $(MICRO_BINS)

docs/api/wasm-component.md

@@ -86,6 +86,14 @@ waitable-set/callback bindings that compile for wasm32-wasip3 with
 wasi-sdk 34; jco surfaces custom host imports as plain ESM imports, which
 is how the smoke (and any bundler consumer) wires `sendChunk`.
 
+The npm adapter exposes this world as `handle.stream(init, onChunk)`:
+`createStreamHost()` from `cwist-wasm/component` is the `c4punks:cwist/host`
+import, and binding the same object as `streamHost` routes each chunk to
+the callback of the stream in flight. A failing callback reaches the guest
+as the `err` arm of `send-chunk`, which aborts the pump, and `stream()`
+rejects with the callback's own error. `make component-stream-smoke-p3`
+drives the guest through the adapter, including that abort path.
+
 Only wasm32-wasip3 builds this world: 0.2 components cannot lower async
 imports. Bindings are generated per world (`wit-bindgen --world`), and
 `wasm-tools component embed` picks the world explicitly
@@ -141,7 +149,9 @@ overhead is what crosses the default limit.
    Emscripten stays supported regardless.
 3. **0.3 cutover (conditional).** Both preconditions held up under
    evaluation and the streaming boundary has landed in
-   `cwist-guest-stream`. Remaining before the cutover: track JSPI
+   `cwist-guest-stream`, with a JS entry point in the npm adapter
+   (`handle.stream()`, the counterpart of `Module.cwistStreamChunk` on
+   the Emscripten path). Remaining before the cutover: track JSPI
    shipping unflagged in node and browsers, and port the SSE examples
    off the EM_JS callback. Only then does the Emscripten build leave CI.
 

tests/wasm_component_stream_host.js

@@ -3,13 +3,12 @@
  * 'c4punks:cwist/host' to this file when bundling the transpiled stream
  * guest, which is how a bundler consumer wires a custom host import.
  *
- * Chunks are collected on globalThis so the test driver can read them
- * through its own (esbuild-bundled) copy of this module. */
+ * The import is the cwist-wasm adapter's createStreamHost(); the host
+ * object is published on globalThis so the test driver can bind its
+ * handle to the same instance the bundled guest calls into. */
+import { createStreamHost } from '../wasm/npm/component.js';
 
-export const chunks = [];
-globalThis.__cwistStreamChunks = chunks;
+const host = createStreamHost();
+globalThis.__cwistStreamHost = host;
 
-export async function sendChunk(chunk) {
-    /* chunk arrives as a Uint8Array copy owned by the canonical ABI. */
-    chunks.push(Buffer.from(chunk));
-}
+export const sendChunk = host.sendChunk;