From 64b4c05357fbe1779a45b80333addc99191be6eb Mon Sep 17 00:00:00 2001 From: anux Date: Thu, 22 Oct 2026 18:13:55 +0530 Subject: [PATCH 1/2] feat(wasm): streaming dispatch in the component adapter (issue #203) (#426) * feat(wasm): streaming dispatch in the component adapter (issue #203) The cwist-guest-stream world had no JS entry point: the stream smoke called guestStream.dispatchStream directly and collected chunks in a hand-written host import. Consumers of cwist-wasm/component had nothing to move SSE code onto from the Emscripten Module.cwistStreamChunk path. createStreamHost() returns the c4punks:cwist/host import, and handle.stream(init, onChunk) routes each send-chunk to the callback of the stream in flight. The guest awaits each callback (backpressure). A throwing callback reaches the guest as the err arm of send-chunk, which aborts the pump, and stream() rejects with the callback's own error. Streams on one handle run one at a time and wait for a pending useSession(). component.d.ts now says handle() returns a Promise against 0.3 guests, and types the streaming additions. The stream smoke drives the guest through the adapter and adds the abort path. Refs #203 * fix(wasm): share the stream runner registry and ship the io worker - component.js kept STREAM_RUNNERS in module scope, but a bundler consumer (and the stream smoke's esbuild alias) inlines a second copy of the adapter into the guest bundle: the host registers with the bundled copy while stream() looks up the npm copy, so streaming always failed with 'streaming needs streamHost'. Keep the registry on globalThis so every copy shares it. - preview3-shim's preview2-shim io worker spawns new Worker(new URL('./worker-thread.bundle.js', import.meta.url)); esbuild does not emit that file, so component-stream-smoke copies it next to the bundle. --------- Co-authored-by: Lee Yunjin --- Makefile | 5 ++ docs/api/wasm-component.md | 12 +++- tests/wasm_component_stream_host.js | 15 +++-- tests/wasm_component_stream_test.js | 59 +++++++++++++----- wasm/npm/README.md | 36 ++++++++++- wasm/npm/component.d.ts | 39 +++++++++++- wasm/npm/component.js | 94 ++++++++++++++++++++++++++++- 7 files changed, 230 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index 0ae79ca10..ff0d3ab3f 100644 --- a/Makefile +++ b/Makefile @@ -638,6 +638,11 @@ component-stream-smoke: $(COMPONENT_BUILD_DIR)/stream_guest.component.wasm --alias:c4punks:cwist/host=$(abspath tests/wasm_component_stream_host.js) \ --external:@bytecodealliance/jco-node-fs* \ --outfile=$(STREAM_JCO_DIR)/bundle.mjs + # preview3-shim pulls in preview2-shim's io worker, which spawns + # new Worker(new URL("./worker-thread.bundle.js", import.meta.url)); + # esbuild does not emit that file, so place it next to the bundle. + cp $(STREAM_JCO_DIR)/node_modules/@bytecodealliance/preview2-shim/dist/io/worker-thread.bundle.js \ + $(STREAM_JCO_DIR)/worker-thread.bundle.js BUNDLE=$(abspath $(STREAM_JCO_DIR)/bundle.mjs) \ $(NODE) --experimental-wasm-jspi tests/wasm_component_stream_test.js diff --git a/docs/api/wasm-component.md b/docs/api/wasm-component.md index 8393aca64..b39fbb83d 100644 --- a/docs/api/wasm-component.md +++ b/docs/api/wasm-component.md @@ -86,6 +86,14 @@ waitable-set/callback bindings that compile for wasm32-wasip3 with wasi-sdk 34; jco surfaces custom host imports as plain ESM imports, which is how the smoke (and any bundler consumer) wires `sendChunk`. +The npm adapter exposes this world as `handle.stream(init, onChunk)`: +`createStreamHost()` from `cwist-wasm/component` is the `c4punks:cwist/host` +import, and binding the same object as `streamHost` routes each chunk to +the callback of the stream in flight. A failing callback reaches the guest +as the `err` arm of `send-chunk`, which aborts the pump, and `stream()` +rejects with the callback's own error. `make component-stream-smoke-p3` +drives the guest through the adapter, including that abort path. + Only wasm32-wasip3 builds this world: 0.2 components cannot lower async imports. Bindings are generated per world (`wit-bindgen --world`), and `wasm-tools component embed` picks the world explicitly @@ -141,7 +149,9 @@ overhead is what crosses the default limit. Emscripten stays supported regardless. 3. **0.3 cutover (conditional).** Both preconditions held up under evaluation and the streaming boundary has landed in - `cwist-guest-stream`. Remaining before the cutover: track JSPI + `cwist-guest-stream`, with a JS entry point in the npm adapter + (`handle.stream()`, the counterpart of `Module.cwistStreamChunk` on + the Emscripten path). Remaining before the cutover: track JSPI shipping unflagged in node and browsers, and port the SSE examples off the EM_JS callback. Only then does the Emscripten build leave CI. diff --git a/tests/wasm_component_stream_host.js b/tests/wasm_component_stream_host.js index 80d735026..27edff4f7 100644 --- a/tests/wasm_component_stream_host.js +++ b/tests/wasm_component_stream_host.js @@ -3,13 +3,12 @@ * 'c4punks:cwist/host' to this file when bundling the transpiled stream * guest, which is how a bundler consumer wires a custom host import. * - * Chunks are collected on globalThis so the test driver can read them - * through its own (esbuild-bundled) copy of this module. */ + * The import is the cwist-wasm adapter's createStreamHost(); the host + * object is published on globalThis so the test driver can bind its + * handle to the same instance the bundled guest calls into. */ +import { createStreamHost } from '../wasm/npm/component.js'; -export const chunks = []; -globalThis.__cwistStreamChunks = chunks; +const host = createStreamHost(); +globalThis.__cwistStreamHost = host; -export async function sendChunk(chunk) { - /* chunk arrives as a Uint8Array copy owned by the canonical ABI. */ - chunks.push(Buffer.from(chunk)); -} +export const sendChunk = host.sendChunk; diff --git a/tests/wasm_component_stream_test.js b/tests/wasm_component_stream_test.js index cdcfe3926..734c579fa 100644 --- a/tests/wasm_component_stream_test.js +++ b/tests/wasm_component_stream_test.js @@ -1,8 +1,8 @@ /* Drives the jco-transpiled cwist-guest-stream component (bundled by * `make component-stream-smoke` with tests/wasm_component_stream_host.js - * aliased in as the c4punks:cwist/host import) and asserts that a - * streaming dispatch delivers the serialized response head-first through - * the async host send-chunk. + * aliased in as the c4punks:cwist/host import) through the cwist-wasm + * component adapter, and asserts that handle.stream() delivers the + * serialized response head-first through the async host send-chunk. * * Run under node's JSPI: node --experimental-wasm-jspi * tests/wasm_component_stream_test.js with BUNDLE pointing at the esbuild @@ -10,7 +10,7 @@ 'use strict'; const path = require('path'); -const { buildRequestBytes } = require(path.join(__dirname, '..', 'wasm', 'npm', 'index.js')); +const { createCwistFromComponent } = require(path.join(__dirname, '..', 'wasm', 'npm', 'component.js')); const BUNDLE = process.env.BUNDLE; if (!BUNDLE) { @@ -24,21 +24,28 @@ if (!BUNDLE) { /* Command component: run() executes main() (route registration). */ await component.run.run(); + const handle = createCwistFromComponent({ + dispatch: component.guest.dispatch, + useSession: component.guest.useSession, + dispatchStream: component.guestStream.dispatchStream, + streamHost: globalThis.__cwistStreamHost, + }); + /* Sync WIT exports of a 0.3 component lower to async functions under * jco, same as the plain guest. */ - const resBytes = await component.guest.dispatch( - buildRequestBytes({ method: 'GET', path: '/hello' })); - const head = new TextDecoder().decode(resBytes); - if (!head.startsWith('HTTP/1.1 200')) { - throw new Error('plain dispatch broken in stream world: ' + head.slice(0, 40)); + const res = await handle({ method: 'GET', path: '/hello' }); + if (res.status !== 200) { + throw new Error('plain dispatch broken in stream world: ' + res.status); } /* Streaming dispatch: the whole serialized response arrives through - * host.send-chunk, head first, body last. */ - const chunks = globalThis.__cwistStreamChunks; - chunks.length = 0; - await component.guestStream.dispatchStream( - buildRequestBytes({ method: 'GET', path: '/events' })); + * host.send-chunk, head first, body last. The callback awaits, so the + * guest's pump waits on it (backpressure). */ + const chunks = []; + await handle.stream({ method: 'GET', path: '/events' }, async (chunk) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + chunks.push(Buffer.from(chunk)); + }); if (chunks.length < 2) { throw new Error('expected head + body chunks, got ' + chunks.length); @@ -54,6 +61,30 @@ if (!BUNDLE) { throw new Error('missing SSE body: ' + JSON.stringify(wire.slice(-80))); } + /* A failing callback aborts the dispatch, and stream() rejects with the + * callback's own error rather than the guest's dispatch-error. */ + const boom = new Error('client went away'); + let seen = 0; + let rejected = null; + try { + await handle.stream({ method: 'GET', path: '/events' }, () => { + seen++; + throw boom; + }); + } catch (e) { + rejected = e; + } + if (rejected !== boom) { + throw new Error('expected the callback error, got: ' + rejected); + } + if (seen !== 1) { + throw new Error('dispatch kept pumping after the callback failed: ' + seen + ' chunks'); + } + + /* The instance stays usable after an aborted stream. */ + const after = await handle({ method: 'GET', path: '/hello' }); + if (after.status !== 200) throw new Error('dispatch after aborted stream: ' + after.status); + console.log('component-stream-smoke: PASS (streamed /events through host.send-chunk, ' + chunks.length + ' chunks)'); process.exit(0); diff --git a/wasm/npm/README.md b/wasm/npm/README.md index 76383ed21..c3bd380ff 100644 --- a/wasm/npm/README.md +++ b/wasm/npm/README.md @@ -76,11 +76,41 @@ for browsers as-is (verified with esbuild in the CWIST repo, returns a Promise and `await` is required; 0.2 guests stay synchronous. See `docs/api/wasm-component.md` in the CWIST repo for building the guest. +### Streaming (0.3 guests) + +A guest built for the `cwist-guest-stream` world (wasm32-wasip3) streams a +response through its `c4punks:cwist/host` import instead of returning it. +`createStreamHost()` returns that import. Wire it in when you instantiate or +bundle the component, and give the same object to the adapter: + +```js +const { createCwistFromComponent, createStreamHost } = require('cwist-wasm/component'); + +const host = createStreamHost(); // { sendChunk }: the c4punks:cwist/host import +// ...instantiate or bundle the transpiled guest with `host` as that import... +const handle = createCwistFromComponent({ + dispatch: guest.dispatch, + useSession: guest.useSession, + dispatchStream: guestStream.dispatchStream, + streamHost: host, +}); + +await handle.stream({ method: 'GET', path: '/events' }, async (chunk) => { + // chunk is a Uint8Array: the serialized head first, then body slices + await writer.write(chunk); +}); +``` + +The guest waits for each callback, so a callback that returns a Promise +applies backpressure. If the callback throws or rejects, the dispatch stops +and `stream()` rejects with that error. Streaming dispatches on one handle +run one at a time. + ## Scope This wrapper is intentionally thin: routing, middleware, validation, and rendering all stay in your C handlers. It only serializes requests into HTTP/1.1 bytes, calls `cwist_app_dispatch_memory()` inside WASM, and parses -the serialized response back into JS objects. Streaming and session -persistence are not handled here; see `docs/api/wasm.md` in the CWIST repo -for the current caveats. +the serialized response back into JS objects. Session persistence is not +handled here, and streaming is only available on the component path above; +see `docs/api/wasm.md` in the CWIST repo for the current caveats. diff --git a/wasm/npm/component.d.ts b/wasm/npm/component.d.ts index 5077effd4..ca73bb005 100644 --- a/wasm/npm/component.d.ts +++ b/wasm/npm/component.d.ts @@ -15,23 +15,50 @@ export interface CwistResponse { body: Uint8Array; } +/** + * Receives one serialized chunk of a streaming response: the head (status + * line + headers) first, then body slices. Return a Promise to apply + * backpressure; a throw or rejection aborts the dispatch. + */ +export type CwistChunkCallback = (chunk: Uint8Array) => void | Promise; + export interface CwistHandle { - (init?: CwistRequestInit): CwistResponse; + /** + * Synchronous against a 0.2 (wasip2) guest; returns a Promise against a + * 0.3 (wasip3) guest, whose exports jco lowers to async functions. + */ + (init?: CwistRequestInit): CwistResponse | Promise; /** * Pin the session signing secret (host-injected contract; issue #93). * Pass null to let CWIST generate a per-instance random secret (dev * mode). Must be called before the first session-bearing dispatch. */ useSession(secret: string | null): void; + /** + * Streaming dispatch through the cwist-guest-stream world (0.3 only): + * resolves once the last chunk has been delivered to onChunk, rejects + * with the dispatch-error or with the callback's own error. Needs + * dispatchStream and streamHost on the bound component. + */ + stream(init: CwistRequestInit | undefined, onChunk: CwistChunkCallback): Promise; +} + +/** The c4punks:cwist/host import of the cwist-guest-stream world. */ +export interface CwistStreamHost { + sendChunk(chunk: Uint8Array): Promise; } /** Minimal shape of the guest interface of a jco-transpiled cwist-guest * component (docs/api/wasm-component.md, issue #203). */ export interface CwistComponent { - dispatch(request: Uint8Array): Uint8Array; - useSession(secret: string | null): void; + dispatch(request: Uint8Array): Uint8Array | Promise; + useSession(secret: string | null): void | Promise; /** Optional declarative session secret, applied once at binding time. */ cwistSessionSecret?: string; + /** guest-stream's dispatch-stream; only cwist-guest-stream guests have it. */ + dispatchStream?(request: Uint8Array): Promise; + /** The createStreamHost() object wired as the component's host import. */ + streamHost?: CwistStreamHost; } /** @@ -40,3 +67,9 @@ export interface CwistComponent { * dispatch-error variant. */ export function createCwistFromComponent(component: CwistComponent): CwistHandle; + +/** + * Create the c4punks:cwist/host import for a cwist-guest-stream component; + * pass the same object as `streamHost` when binding the component. + */ +export function createStreamHost(): CwistStreamHost; diff --git a/wasm/npm/component.js b/wasm/npm/component.js index 105d443fd..064aa4319 100644 --- a/wasm/npm/component.js +++ b/wasm/npm/component.js @@ -11,11 +11,78 @@ * const res = handle({ method: 'GET', path: '/hello' }); * // res.status, res.headers, res.body (Uint8Array) — same as createCwist() * // Against a 0.3 (wasip3) component handle() returns a Promise: await it. + * + * Streaming (cwist-guest-stream world, 0.3 only): the guest pushes chunks + * through the c4punks:cwist/host import, which is wired when the component + * is instantiated or bundled. createStreamHost() returns that import; + * handing the same host to the adapter routes each chunk to the callback + * of the handle.stream() call in flight: + * + * const host = createStreamHost(); // { sendChunk } -> host import + * const handle = createCwistFromComponent({ ...guest, + * dispatchStream: guestStream.dispatchStream, streamHost: host }); + * await handle.stream({ path: '/events' }, (chunk) => { ... }); */ 'use strict'; const { buildRequestBytes, parseResponseBytes } = require('./index.js'); +/* Internal: the dispatch runner behind each stream host. Kept off the host + * object so the host is exactly the shape of the WIT import. + * + * The registry must live on globalThis: a bundler (esbuild alias, webpack) + * inlines a second copy of this module into the guest bundle when the host + * import references createStreamHost(), while the consumer's adapter comes + * from the npm package — two module instances, two WeakMaps, and the lookup + * in stream() would miss every registration made through the bundled copy. */ +const STREAM_RUNNERS = (globalThis.__cwistStreamRunners ||= new WeakMap()); + +/* jco turns a thrown value carrying `payload` into the err arm of the + * import's result<_, string>; the guest then aborts the dispatch. */ +function streamFailure(message) { + return { payload: message }; +} + +/** + * Create the c4punks:cwist/host import for a cwist-guest-stream component. + * Pass the returned object (or its sendChunk) as the host import, and the + * same object as `streamHost` to createCwistFromComponent(). + * @returns {{sendChunk: (chunk: Uint8Array) => Promise}} + */ +function createStreamHost() { + let sink = null; + let sinkError = null; + + /* Awaited by the guest: delaying resolution is the backpressure. */ + async function sendChunk(chunk) { + if (!sink) throw streamFailure('cwist-wasm: send-chunk outside a streaming dispatch'); + try { + await sink(chunk); + } catch (e) { + sinkError = { error: e }; + throw streamFailure('cwist-wasm: chunk callback failed: ' + String(e && e.message ? e.message : e)); + } + } + + const host = { sendChunk }; + STREAM_RUNNERS.set(host, async function run(onChunk, dispatch) { + sink = onChunk; + sinkError = null; + try { + await dispatch(); + } catch (e) { + /* A failed callback aborts the guest's pump, which then reports a + * dispatch-error; surface the callback's own error instead. */ + if (sinkError) throw sinkError.error; + throw e; + } finally { + sink = null; + } + if (sinkError) throw sinkError.error; + }); + return host; +} + /** * Create a CWIST request handler bound to an instantiated cwist-guest * component (jco transpile output of a guest built for wit/cwist.wit). @@ -71,8 +138,33 @@ function createCwistFromComponent(component) { return parseResponseBytes(component.dispatch(request)); } + /* One sink per host, so streaming dispatches run one at a time; they also + * wait for any pending useSession(). */ + let streamQueue = Promise.resolve(); + + function stream(init, onChunk) { + if (typeof component.dispatchStream !== 'function') { + throw new Error( + 'cwist-wasm: component does not expose dispatchStream; build the guest for the cwist-guest-stream world (wasm32-wasip3)' + ); + } + const run = component.streamHost && STREAM_RUNNERS.get(component.streamHost); + if (!run) { + throw new Error( + 'cwist-wasm: streaming needs streamHost, the createStreamHost() object wired as the c4punks:cwist/host import' + ); + } + if (typeof onChunk !== 'function') throw new Error('cwist-wasm: stream() needs a chunk callback'); + const request = buildRequestBytes(init || {}); + const result = Promise.all([chain, streamQueue]) + .then(() => run(onChunk, () => component.dispatchStream(request))); + streamQueue = result.then(() => {}, () => {}); + return result; + } + handle.useSession = useSession; + handle.stream = stream; return handle; } -module.exports = { createCwistFromComponent }; +module.exports = { createCwistFromComponent, createStreamHost }; From 050e3141028b7638fc8d82e190ccbabba7eca4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9D=B4=EC=9C=A4=EC=A7=84=28Lee=20Yunjin=29?= <168180007+gg582@users.noreply.github.com> Date: Fri, 23 Oct 2026 00:16:19 +0900 Subject: [PATCH 2/2] feat(build): CWIST_FAT/CWIST_APP_FAT embedded-friendly fat archive (#427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(build): CWIST_FAT/CWIST_APP_FAT embedded-friendly fat archive - make CWIST_FAT=1 folds every vendored static archive (BoringSSL, lsquic, libttak, cnats, cJSON, uriparser, usrsctp) into libcwist.a via an ar MRI merge (libtool -static on Darwin), so copying the single archive plus include/cwist onto an embedded target is enough to link a server app; only system libraries stay external. A .build-fat stamp re-archives when the toggle flips, without recompiling objects. - CWIST_APP_FAT=1 implies CWIST_FAT=1 and drops the vendored archives from the example apps' link line (APP_LIBS): the app binary pulls everything from libcwist.a. - New fat.yml workflow builds the fat archive, checks vendored members and key symbols are folded in, links and serves the hello example in a scratch dir with only libcwist.a + headers + app source present, and builds an example with CWIST_APP_FAT=1. * fix(build): select the fat recipe at recipe time and stage fat-dist - The ifeq-selected recipe was chosen at parse time, so fat-dist's target-specific CWIST_FAT=1 built a plain archive with a fat stamp; the embedded link then missed BoringSSL (SSL_get_error via the system OpenSSL DSO). Branch on $(CWIST_FAT) inside the recipe instead. - New fat-dist target stages dist/cwist-fat/: the fat archive plus the vendored headers the public API references (cjson, openssl, sqlite3, ttak) — the bundle copied to the embedded board. - fat.yml now builds fat-dist and runs the link smoke from the bundle. --- .github/workflows/fat.yml | 83 ++++++++++++++++++++++++++ Makefile | 120 ++++++++++++++++++++++++++++---------- 2 files changed, 172 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/fat.yml diff --git a/.github/workflows/fat.yml b/.github/workflows/fat.yml new file mode 100644 index 000000000..0d5a8b4ad --- /dev/null +++ b/.github/workflows/fat.yml @@ -0,0 +1,83 @@ +name: "Fat archive" + +# CWIST_FAT=1 folds the vendored static archives into libcwist.a and +# `make fat-dist` stages it with the headers the public API pulls in, so an +# embedded target needs only the bundle plus the app source at link time; +# CWIST_APP_FAT=1 links the example apps against that fat archive alone. +# This job builds the bundle, proves a server app links and serves with +# nothing but the bundle copied next to its source, and builds an example +# with CWIST_APP_FAT=1. + +on: + push: + branches: [main, master, dev] + pull_request: + branches: [main, master, dev] + workflow_dispatch: + +permissions: + contents: read + +jobs: + fat: + name: Fat archive build + embedded-style link smoke + if: github.event_name != 'push' || !contains(github.event.head_commit.message, '[skip ci]') + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake libcurl4-openssl-dev libnghttp2-dev libbrotli-dev libzstd-dev + + - name: Build the fat archive and embedded bundle + run: make fat-dist -j"$(nproc)" + + - name: Verify vendored archives are folded in + run: | + # Members of every vendored archive must be present, not just the + # CWIST objects: the archive alone has to satisfy an app link. + ar t libcwist.a | grep -q '^cJSON\.o$' + ar t libcwist.a | grep -q '^bio_ssl\.cc\.o$' + ar t libcwist.a | grep -q '^lsqpack\.c\.o$' + ar t libcwist.a | grep -q '^UriCommon\.c\.o$' + nm --defined-only libcwist.a | grep -qw 'T SSL_new' + nm --defined-only libcwist.a | grep -qw 'T lsquic_engine_new' + nm --defined-only libcwist.a | grep -qw 'T natsConnection_Connect' + nm --defined-only libcwist.a | grep -qw 'T sqlite3_open' + + - name: Embedded-style link smoke (bundle + app source only) + run: | + # Mirror the embedded workflow: copy just the fat-dist bundle and + # the app source onto the target, link against system libs only. + work=$(mktemp -d) + cp -r dist/cwist-fat "$work/bundle" + cp example/micro/01-hello/main.c "$work/" + cd "$work" + "${CC:-gcc}" -O2 -std=gnu17 -Ibundle/include -o hello main.c \ + bundle/lib/libcwist.a \ + $(pkg-config --libs libcurl libnghttp2 libbrotlienc libbrotlicommon libbrotlidec libzstd) \ + -pthread -ldl -lm -lstdc++ -lz + ./hello & + pid=$! + sleep 2 + test "$(curl -s -m 3 http://127.0.0.1:8080/)" = "Hello, CWIST!" + kill "$pid" + + - name: CWIST_APP_FAT example build + run: | + # The app link must reference libcwist.a and no vendored archive. + out=$(make CWIST_APP_FAT=1 example/micro/01-hello/hello 2>&1 | tail -1) + echo "$out" + echo "$out" | grep -q 'libcwist\.a' + ! echo "$out" | grep -q 'lib/[a-z0-9_/-]*/[^ ]*\.a' + ./example/micro/01-hello/hello & + pid=$! + sleep 2 + test "$(curl -s -m 3 http://127.0.0.1:8080/)" = "Hello, CWIST!" + kill "$pid" diff --git a/Makefile b/Makefile index ff0d3ab3f..a4ff9b443 100644 --- a/Makefile +++ b/Makefile @@ -331,6 +331,7 @@ SRCS = src/core/sstring/sstring.c \ EMCC ?= emcc EMAR ?= emar NODE ?= node +RANLIB ?= ranlib WASM_BUILD_DIR = .wasm-build WASM_SRCS = src/core/sstring/sstring.c \ src/core/seq/seq.c \ @@ -686,6 +687,21 @@ component-browser-test: component-browser-smoke # Object Files and Target OBJS = $(SRCS:.c=.o) LIB_NAME = libcwist.a + +# Rebuild every object when the compile flags change. CWIST_WEBTRANSPORT adds +# and removes struct members in http3.c, so flipping it without a clean left a +# library holding objects built both ways: the layouts disagreed and the result +# was a stack smash at runtime, not a compile error. .build-flags records the +# flags and every object depends on it, so a changed flag forces a rebuild. +.build-flags: FORCE + @printf '%s' '$(CFLAGS)' | cmp -s - $@ || printf '%s' '$(CFLAGS)' > $@ +# Records the fat-archive toggle so flipping CWIST_FAT re-archives libcwist.a +# without forcing a recompile of unchanged objects. +.build-fat: FORCE + @printf '%s' 'fat=$(CWIST_FAT)' | cmp -s - $@ || printf '%s' 'fat=$(CWIST_FAT)' > $@ +FORCE: +.PHONY: FORCE fat-dist +$(OBJS): .build-flags LIBTTAK_DIR = lib/libttak LIBTTAK_LIB = $(LIBTTAK_DIR)/lib/libttak.a LIBTTAK_EXTRA_CFLAGS = @@ -743,6 +759,19 @@ ifeq ($(CWIST_WEBRTC),1) LIBS += $(USRSCTP_LIB) endif +# --- Fat (embedded-friendly) build options ----------------------------------- +# CWIST_FAT=1 folds every vendored static archive into libcwist.a, so copying +# the single archive plus the CWIST headers onto an embedded target is enough +# to link a server app; only system libraries stay external. +# CWIST_APP_FAT=1 builds the example apps against the fat archive alone (the +# vendored archives drop off their link line); it implies CWIST_FAT=1. +ifeq ($(CWIST_APP_FAT),1) +CWIST_FAT = 1 +APP_LIBS = $(filter-out $(EXTERNAL_LIBS),$(LIBS)) +else +APP_LIBS = $(LIBS) +endif + # --- Build Targets --- all: $(LIBTTAK_LIB) $(CJSON_LIB) $(URIPARSER_LIB) $(SQLITE_DIR)/sqlite3.c $(LSQUIC_LIB) $(LIB_NAME) @@ -794,10 +823,39 @@ lib/lsquic/include/lsquic.h: git submodule update --init --recursive $(LSQUIC_DIR); \ fi -$(LIB_NAME): $(EXTERNAL_LIBS) $(OBJS) - @echo "Creating thin static library..." +$(LIB_NAME): .build-fat $(EXTERNAL_LIBS) $(OBJS) @rm -f $@ - ar rcs $@ $(OBJS) + @if [ "$(CWIST_FAT)" = "1" ]; then \ + echo "Creating fat static library (CWIST objects + vendored archives)..."; \ + if [ "$(UNAME_S)" = "Darwin" ]; then \ + libtool -static -o $@ $(sort $(OBJS)) $(EXTERNAL_LIBS); \ + else \ + { echo "CREATE $@"; \ + for o in $(sort $(OBJS)); do echo "ADDMOD $$o"; done; \ + for l in $(EXTERNAL_LIBS); do echo "ADDLIB $$l"; done; \ + echo "SAVE"; echo "END"; } | $(AR) -M && $(RANLIB) $@; \ + fi; \ + else \ + echo "Creating static library..."; \ + $(AR) rcs $@ $(OBJS); \ + fi + +# The embedded bundle: the fat archive plus every header the public API +# pulls in (the public headers reference cjson, openssl, sqlite3 and ttak), +# laid out as the compiler expects. Copy FAT_DIST_DIR onto the target board +# and build apps with -I/include /lib/libcwist.a; only system +# libraries stay external. +FAT_DIST_DIR ?= dist/cwist-fat +fat-dist: CWIST_FAT = 1 +fat-dist: $(LIB_NAME) + rm -rf $(FAT_DIST_DIR) + mkdir -p $(FAT_DIST_DIR)/lib $(FAT_DIST_DIR)/include/cjson + cp $(LIB_NAME) $(FAT_DIST_DIR)/lib/ + cp -R include/cwist $(FAT_DIST_DIR)/include/ + cp $(CJSON_DIR)/cJSON.h $(FAT_DIST_DIR)/include/cjson/ + cp -R $(LIBTTAK_DIR)/include/ttak $(FAT_DIST_DIR)/include/ + cp -R $(BORINGSSL_DIR)/include/openssl $(FAT_DIST_DIR)/include/ + cp $(SQLITE_DIR)/sqlite3.h $(FAT_DIST_DIR)/include/ $(LIBTTAK_LIB): @echo "Building libttak..." @@ -1548,37 +1606,37 @@ EXAMPLE_BINS = example/simple-server/simple-server \ examples: $(EXAMPLE_BINS) example/simple-server/simple-server: $(LIB_NAME) example/simple-server/main.c - $(CC) $(CFLAGS) -o $@ example/simple-server/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/simple-server/main.c $(LIB_NAME) $(APP_LIBS) example/http/step-1-hello-world/hello-world: $(LIB_NAME) example/http/step-1-hello-world/main.c - $(CC) $(CFLAGS) -o $@ example/http/step-1-hello-world/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/http/step-1-hello-world/main.c $(LIB_NAME) $(APP_LIBS) example/http/step-2-mux-router/mux-router: $(LIB_NAME) example/http/step-2-mux-router/main.c - $(CC) $(CFLAGS) -o $@ example/http/step-2-mux-router/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/http/step-2-mux-router/main.c $(LIB_NAME) $(APP_LIBS) example/http/step-3-query-params/query-params: $(LIB_NAME) example/http/step-3-query-params/main.c - $(CC) $(CFLAGS) -o $@ example/http/step-3-query-params/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/http/step-3-query-params/main.c $(LIB_NAME) $(APP_LIBS) example/http/step-4-json-api/json-api: $(LIB_NAME) example/http/step-4-json-api/main.c - $(CC) $(CFLAGS) -o $@ example/http/step-4-json-api/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/http/step-4-json-api/main.c $(LIB_NAME) $(APP_LIBS) example/jwt/step-2-http-auth/jwt-auth: $(LIB_NAME) example/jwt/step-2-http-auth/main.c - $(CC) $(CFLAGS) -o $@ example/jwt/step-2-http-auth/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/jwt/step-2-http-auth/main.c $(LIB_NAME) $(APP_LIBS) example/cde-json-viewer/cde-json-viewer: $(LIB_NAME) example/cde-json-viewer/main.c - $(CC) $(CFLAGS) -o $@ example/cde-json-viewer/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/cde-json-viewer/main.c $(LIB_NAME) $(APP_LIBS) example/db/step-1-open-query/open-query: $(LIB_NAME) example/db/step-1-open-query/main.c - $(CC) $(CFLAGS) -o $@ example/db/step-1-open-query/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/db/step-1-open-query/main.c $(LIB_NAME) $(APP_LIBS) example/db/step-2-migrations/migrations: $(LIB_NAME) example/db/step-2-migrations/main.c - $(CC) $(CFLAGS) -o $@ example/db/step-2-migrations/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/db/step-2-migrations/main.c $(LIB_NAME) $(APP_LIBS) example/db/step-4-json-insert/json-insert: $(LIB_NAME) example/db/step-4-json-insert/main.c - $(CC) $(CFLAGS) -o $@ example/db/step-4-json-insert/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/db/step-4-json-insert/main.c $(LIB_NAME) $(APP_LIBS) example/rps-showcase/rps-showcase: $(LIB_NAME) example/rps-showcase/main.c - $(CC) $(CFLAGS) -o $@ example/rps-showcase/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/rps-showcase/main.c $(LIB_NAME) $(APP_LIBS) # Micro examples MICRO_BINS = example/micro/01-hello/hello \ @@ -1626,55 +1684,55 @@ examples-check: $(LIB_NAME) $(EXAMPLE_BINS) $(MICRO_BINS) done example/micro/01-hello/hello: $(LIB_NAME) example/micro/01-hello/main.c - $(CC) $(CFLAGS) -o $@ example/micro/01-hello/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/01-hello/main.c $(LIB_NAME) $(APP_LIBS) example/micro/02-routes/routes: $(LIB_NAME) example/micro/02-routes/main.c - $(CC) $(CFLAGS) -o $@ example/micro/02-routes/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/02-routes/main.c $(LIB_NAME) $(APP_LIBS) example/micro/03-path-params/path-params: $(LIB_NAME) example/micro/03-path-params/main.c - $(CC) $(CFLAGS) -o $@ example/micro/03-path-params/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/03-path-params/main.c $(LIB_NAME) $(APP_LIBS) example/micro/04-query-params/query-params: $(LIB_NAME) example/micro/04-query-params/main.c - $(CC) $(CFLAGS) -o $@ example/micro/04-query-params/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/04-query-params/main.c $(LIB_NAME) $(APP_LIBS) example/micro/05-json/json: $(LIB_NAME) example/micro/05-json/main.c - $(CC) $(CFLAGS) -o $@ example/micro/05-json/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/05-json/main.c $(LIB_NAME) $(APP_LIBS) example/micro/06-orm-insert/orm-insert: $(LIB_NAME) example/micro/06-orm-insert/main.c - $(CC) $(CFLAGS) -o $@ example/micro/06-orm-insert/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/06-orm-insert/main.c $(LIB_NAME) $(APP_LIBS) example/micro/07-orm-query/orm-query: $(LIB_NAME) example/micro/07-orm-query/main.c - $(CC) $(CFLAGS) -o $@ example/micro/07-orm-query/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/07-orm-query/main.c $(LIB_NAME) $(APP_LIBS) example/micro/08-orm-update-delete/orm-update-delete: $(LIB_NAME) example/micro/08-orm-update-delete/main.c - $(CC) $(CFLAGS) -o $@ example/micro/08-orm-update-delete/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/08-orm-update-delete/main.c $(LIB_NAME) $(APP_LIBS) example/micro/09-html-builder/html-builder: $(LIB_NAME) example/micro/09-html-builder/main.c - $(CC) $(CFLAGS) -o $@ example/micro/09-html-builder/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/09-html-builder/main.c $(LIB_NAME) $(APP_LIBS) example/micro/10-static-files/static-files: $(LIB_NAME) example/micro/10-static-files/main.c - $(CC) $(CFLAGS) -o $@ example/micro/10-static-files/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/10-static-files/main.c $(LIB_NAME) $(APP_LIBS) example/micro/11-jwt-auth/jwt-auth: $(LIB_NAME) example/micro/11-jwt-auth/main.c - $(CC) $(CFLAGS) -o $@ example/micro/11-jwt-auth/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/11-jwt-auth/main.c $(LIB_NAME) $(APP_LIBS) example/micro/12-middleware/middleware: $(LIB_NAME) example/micro/12-middleware/main.c - $(CC) $(CFLAGS) -o $@ example/micro/12-middleware/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/12-middleware/main.c $(LIB_NAME) $(APP_LIBS) example/micro/13-nuke-db/nuke-db: $(LIB_NAME) example/micro/13-nuke-db/main.c - $(CC) $(CFLAGS) -o $@ example/micro/13-nuke-db/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/13-nuke-db/main.c $(LIB_NAME) $(APP_LIBS) example/micro/14-websocket/websocket: $(LIB_NAME) example/micro/14-websocket/main.c - $(CC) $(CFLAGS) -o $@ example/micro/14-websocket/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/14-websocket/main.c $(LIB_NAME) $(APP_LIBS) example/micro/15-pqc-tls/pqc-tls: $(LIB_NAME) example/micro/15-pqc-tls/main.c - $(CC) $(CFLAGS) -o $@ example/micro/15-pqc-tls/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/15-pqc-tls/main.c $(LIB_NAME) $(APP_LIBS) example/micro/16-rdbms-auto/rdbms-auto: $(LIB_NAME) example/micro/16-rdbms-auto/main.c - $(CC) $(CFLAGS) -o $@ example/micro/16-rdbms-auto/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/16-rdbms-auto/main.c $(LIB_NAME) $(APP_LIBS) example/micro/17-blog-crud/blog-crud: $(LIB_NAME) example/micro/17-blog-crud/main.c - $(CC) $(CFLAGS) -o $@ example/micro/17-blog-crud/main.c $(LIB_NAME) $(LIBS) + $(CC) $(CFLAGS) -o $@ example/micro/17-blog-crud/main.c $(LIB_NAME) $(APP_LIBS) clean-examples: rm -f $(EXAMPLE_BINS) $(MICRO_BINS)