From 157aea708b71cdf40625f9f0ab20d6388fdc846f Mon Sep 17 00:00:00 2001 From: Alex Moreton Date: Thu, 2 Jul 2026 16:21:45 +0100 Subject: [PATCH 1/5] Verify-session returns exp + adds purge session cache for session completeness via a db hook with sha256 hashing --- services/api/src/lib/auth.ts | 22 ++++++++++++++++++++ services/api/src/lib/chat-internal.ts | 10 +++++++-- services/api/src/lib/chat-ws-push.ts | 30 +++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/services/api/src/lib/auth.ts b/services/api/src/lib/auth.ts index 3c40a55..2553747 100644 --- a/services/api/src/lib/auth.ts +++ b/services/api/src/lib/auth.ts @@ -1,7 +1,9 @@ +import { createHash } from "node:crypto"; import { betterAuth } from "better-auth"; import { prismaAdapter } from "better-auth/adapters/prisma"; import { bearer } from "better-auth/plugins"; import { prisma } from "@repo/database"; +import { purgeSessionCache } from "./chat-ws-push"; // ── Better Auth server instance ─────────────────────────────────────────────── // Sessions are DB-backed (via Prisma) — fully revocable on logout or ban. // The `bearer` plugin enables Authorization: Bearer for API clients @@ -73,6 +75,26 @@ export const auth = betterAuth({ }, }, }, + // Write-through invalidation of the edge session cache (ADR-0017 §3). Fires + // on sign-out and every revoke path, one place. sha256(session.token) is the + // KV key the Worker wrote (auth.ts sha256Hex — UTF-8 bytes, lowercase hex), + // so the digests match. Fire-and-forget: the row is already deleted here + // (authoritative) and the KV TTL backstops a missed purge, so this must + // never throw or block the deletion. + session: { + delete: { + after: async (session) => { + try { + const hash = createHash("sha256") + .update(session.token) + .digest("hex"); + await purgeSessionCache(hash); + } catch { + // best-effort; TTL bounds revocation lag + } + }, + }, + }, }, session: { diff --git a/services/api/src/lib/chat-internal.ts b/services/api/src/lib/chat-internal.ts index 11feebe..0628ee4 100644 --- a/services/api/src/lib/chat-internal.ts +++ b/services/api/src/lib/chat-internal.ts @@ -2,7 +2,7 @@ // // POST /internal/chat/verify-session // body: { token } -// 200: { userId } +// 200: { userId, exp } exp = session expiry, unix seconds // 401: unauthenticated bearer // // The /internal/chat/persist endpoint was removed in M2 per ADR-010 — the @@ -22,6 +22,10 @@ interface VerifySessionRequest { interface VerifySessionResponse { userId: string; + // Session expiry, unix seconds. Lets the edge cache (ADR-0017) avoid serving + // a session past its real end within the KV TTL window. Additive field — + // older Workers that ignore it are unaffected. + exp: number; } function verifyHmac(request: Request): boolean { @@ -75,7 +79,9 @@ async function handleVerifySession(request: Request): Promise { return new Response("invalid session", { status: 401 }); } - const out: VerifySessionResponse = { userId: session.user.id }; + // expiresAt is a Date on the DB-backed session record; to unix seconds. + const exp = Math.floor(new Date(session.session.expiresAt).getTime() / 1000); + const out: VerifySessionResponse = { userId: session.user.id, exp }; return new Response(JSON.stringify(out), { status: 200, headers: { "content-type": "application/json" }, diff --git a/services/api/src/lib/chat-ws-push.ts b/services/api/src/lib/chat-ws-push.ts index 483f0d0..ecf923b 100644 --- a/services/api/src/lib/chat-ws-push.ts +++ b/services/api/src/lib/chat-ws-push.ts @@ -73,3 +73,33 @@ export async function notifyMlsWelcome( return { ok: false }; } } + +/** + * Evict a session from the edge cache (ADR-0017 §3, write-through invalidation). + * `tokenHash` is the hex sha256(token) — the same key the Worker writes — so the + * raw token never leaves this service. + * + * Best-effort by design: the session row is already deleted (authoritative) and + * the KV TTL backstops a missed purge, so callers MUST NOT block on the result + * or fail sign-out / revoke if this returns `{ ok: false }`. + */ +export async function purgeSessionCache( + tokenHash: string, +): Promise<{ ok: boolean }> { + const url = chatWsUrl(); + if (!url || !tokenHash) return { ok: false }; + + try { + const resp = await fetch(`${url}/internal/session/purge`, { + method: "POST", + headers: { + "content-type": "application/json", + [SECRET_HEADER]: Resource.ChatWsHmacSecret.value, + }, + body: JSON.stringify({ tokenHash }), + }); + return { ok: resp.status === 200 }; + } catch { + return { ok: false }; + } +} From 7942ecc227f21e0d8a1f81dbd80f680fbb4516aa Mon Sep 17 00:00:00 2001 From: Alex Moreton Date: Thu, 2 Jul 2026 16:25:05 +0100 Subject: [PATCH 2/5] Read path + KV cache-aside verify + session cache binding --- infra/stacks/chat-ws.ts | 68 ++++++++-- services/chat-ws/src/auth.ts | 146 +++++++++++++++++++-- services/chat-ws/worker-configuration.d.ts | 11 ++ services/chat-ws/wrangler.jsonc | 10 +- 4 files changed, 212 insertions(+), 23 deletions(-) diff --git a/infra/stacks/chat-ws.ts b/infra/stacks/chat-ws.ts index e1af3a1..d7ee392 100644 --- a/infra/stacks/chat-ws.ts +++ b/infra/stacks/chat-ws.ts @@ -29,6 +29,16 @@ import { databaseUrl } from "./secrets"; // shape may shift between SST versions — verify `sst diff` before first deploy // and tweak the transform if Cloudflare provider rejects the binding format. +// ── Edge session cache (ADR-0017) ──────────────────────────────────────────── +// Cache-aside KV in front of WS-connect session verification. Neon stays +// authoritative; KV is a short-TTL read-through cache keyed by sha256(token), +// value { userId, exp }. Bound raw as `env.SESSION_CACHE` (a namespace binding, +// like the DO namespaces — NOT a linked secret, so it is wired via +// transform.worker.bindings below rather than `link`). The cache-aside read +// path (B1.2+) and the HMAC-gated /internal/session/purge worker (B1.5) are +// separate steps; B1.1 provisions the namespace + binding only. +export const sessionCache = new sst.cloudflare.Kv("SessionCache"); + export const chatWsWorker = new sst.cloudflare.Worker("ChatWs", { handler: "services/chat-ws/src/index.ts", url: true, @@ -48,13 +58,31 @@ export const chatWsWorker = new sst.cloudflare.Worker("ChatWs", { // dedicated IAM user with sns:Publish scoped to this topic only. // Provisioned out-of-band; secrets are set via `sst secret set`. AWS_REGION: $app.providers?.aws?.region ?? "eu-west-1", + // Edge session cache (ADR-0017 §3). TTL bounds worst-case revocation lag; + // 120s is the tuned starting point (60–300s viable). String because Worker + // env vars are strings — the cache layer (B1.2) parses to seconds. Kept + // env-tunable so it can be adjusted under B1.7 load testing without a code + // change. + SESSION_CACHE_TTL: "120", + // Kill-switch (ADR-0017 consequences). "0"/"false" → the Worker falls back + // to origin-only verification (existing Lambda path) with no redeploy. + SESSION_CACHE_ENABLED: "1", + // Load-test metrics (B1.7). "1" → per-verify "SCM" log for `wrangler tail` + // to tally cache hit rate + KV write rate. Keep "0" in prod; flip to "1" + // (+ redeploy) only for a load-test run, then flip back. + SESSION_CACHE_METRICS: "0", }, transform: { worker: (args) => { // Bind the Chat + UserInbox Durable Object namespaces. // SQLite-backed DOs are the recommended path on the Workers Paid plan // (cheaper, supports point-in-time recovery, modern API). - args.bindings = $resolve([args.bindings]).apply(([bindings]) => [ + // Object form of $resolve so each value keeps its own type — the array + // form unifies mixed Output types into a union and widens namespace_id. + args.bindings = $resolve({ + bindings: args.bindings, + sessionCacheId: sessionCache.id, + }).apply(({ bindings, sessionCacheId }) => [ ...(bindings ?? []), { type: "durable_object_namespace", @@ -66,6 +94,20 @@ export const chatWsWorker = new sst.cloudflare.Worker("ChatWs", { name: "USER_INBOX", className: "UserInbox", }, + { + // Edge session cache (ADR-0017). Raw KV namespace binding → + // env.SESSION_CACHE. Not a DO class, so it does NOT touch the + // migration shape below — but adding any binding is still a Worker + // update, which requires the tag bump (see migrations note). + type: "kv_namespace", + name: "SESSION_CACHE", + // MUST be camelCase `namespaceId` — the Pulumi provider's + // WorkersScriptBinding uses camelCase (like `className` above). + // snake_case `namespace_id` is silently dropped → CF 10021 "must have + // a namespace_id". tsc does NOT catch it (excess-property check is + // bypassed through Output.apply). Do not "fix" this back to snake_case. + namespaceId: sessionCacheId, + }, ]); // Migration shape — Pulumi cloudflare provider takes a single object @@ -86,16 +128,22 @@ export const chatWsWorker = new sst.cloudflare.Worker("ChatWs", { // drop classes from newSqliteClasses, and use renamedClasses / // deletedClasses as needed. Do NOT re-declare existing classes in // newSqliteClasses on an update (triggers 10074). - // UPDATE-SHAPE: the script already exists at migration tag v1 with the - // Chat + UserInbox SQLite DO classes created. This deploy changes only - // Worker code (no DO class add/rename/delete), so bump the tag with an - // empty migration and send oldTag so CF's precondition matches the - // deployed v1 (error 10079 "got tag '' when expected v1" = oldTag was - // omitted by the create-shape). Do NOT re-declare existing classes in - // newSqliteClasses (triggers 10074). + // UPDATE-SHAPE (B1.1, ADR-0017): this deploy adds the SESSION_CACHE KV + // binding. That is a Worker-binding change with NO DO class add/rename/ + // delete, so it is an empty migration that only bumps the tag. Send + // oldTag = the currently deployed tag so CF's precondition matches + // (error 10079 "got tag X when expected Y" = oldTag ≠ deployed). Do NOT + // re-declare Chat/UserInbox in newSqliteClasses (triggers 10074). + // + // ⚠ PRECONDITION — deployed tag. This assumes the prior code-only v1→v2 + // deploy (commit 265a35c) actually shipped, i.e. deployed tag == v2. + // VERIFY with `sst diff` before apply. If the v1→v2 deploy never ran + // (e.g. alice/bob was tested on local `wrangler dev`), the deployed tag + // is still v1 — in that case set { oldTag: "v1", newTag: "v2" } here and + // let the KV binding ride that deploy, rather than jumping to v3. args.migrations = { - oldTag: "v1", - newTag: "v2", + oldTag: "v2", + newTag: "v3", }; // Compatibility date with WebSocket auto-reply-to-close behaviour diff --git a/services/chat-ws/src/auth.ts b/services/chat-ws/src/auth.ts index 0630911..5aa8993 100644 --- a/services/chat-ws/src/auth.ts +++ b/services/chat-ws/src/auth.ts @@ -1,13 +1,21 @@ -// Verify a Better Auth bearer token by asking the API Lambda. Cheap on connect -// (one round-trip per WS open) — never per message. +// Verify a Better Auth bearer token on WS connect. // -// HMAC secret comes from `Resource.ChatWsHmacSecret.value` — SST `link`s on -// Cloudflare Workers expose secrets via the Resource module, not as runtime -// env bindings (unlike AWS Lambda). +// Cache-aside edge cache (ADR-0017): check Cloudflare KV first; on a miss verify +// via the API Lambda (Neon-authoritative) and populate KV with a short TTL. +// Neon stays the source of truth — this only takes the DB round-trip off the +// hot path for cache hits. Revocation is bounded by the TTL (backstop) plus the +// write-through purge (/internal/session/purge, added in B1.5). +// +// Invariants: +// - Key is sha256(token) hex, never the raw bearer (a KV dump ≠ credentials). +// - Value is minimal: { userId, exp }. exp guards against serving a session +// that expired inside the TTL window. +// - Negatives are NEVER cached (a failed verify must not poison KV). +// - KV is best-effort: any KV error falls back to origin verification. +// - SESSION_CACHE_ENABLED="0"/"false" is a kill-switch → origin-only, no KV. // -// Future: short-lived JWT minted by Lambda at login + verified at the edge with -// shared HS256 secret would remove this round-trip entirely. Defer until WS -// connect latency is measured under load. +// HMAC secret comes from `Resource.ChatWsHmacSecret.value` — SST `link`s on +// Cloudflare Workers expose secrets via the Resource module, not env bindings. import { Resource } from "sst"; @@ -15,10 +23,62 @@ export interface VerifySessionResult { userId: string; } -export async function verifySession( +// Cached value (ADR-0017 §2). exp = session expiry, unix seconds. +interface CachedSession { + userId: string; + exp: number; +} + +// Origin returns exp too so the cache can store it. exp is null when the origin +// did not send one (e.g. an older Lambda mid-rollout) — auth still succeeds, we +// just skip the cache write, so Worker/Lambda deploy in any order. +interface OriginSession { + userId: string; + exp: number | null; +} + +const TTL_MIN = 60; // KV expirationTtl floor + ADR-0017 §3 lower bound +const TTL_MAX = 300; // ADR-0017 §3 upper bound +const TTL_DEFAULT = 120; + +function cacheEnabled(env: Env): boolean { + const v = env.SESSION_CACHE_ENABLED?.toLowerCase(); + return v !== "0" && v !== "false"; +} + +function cacheTtl(env: Env): number { + const n = Number(env.SESSION_CACHE_TTL); + if (!Number.isFinite(n)) return TTL_DEFAULT; + return Math.min(TTL_MAX, Math.max(TTL_MIN, Math.trunc(n))); +} + +// Load-test instrumentation (B1.7). Off unless SESSION_CACHE_METRICS === "1". +// One compact line per verify, prefixed "SCM" so `wrangler tail | grep SCM` can +// tally hit rate and KV write rate during a run. r = hit | miss | disabled | +// fail; w = whether a KV write happened. Left in, off by default — flip the env +// to "1" for a run, back to "0" after (avoids per-connect log volume in prod). +function metric(env: Env, r: string, wrote = false): void { + if (env.SESSION_CACHE_METRICS === "1") { + console.log(`SCM ${JSON.stringify({ r, w: wrote })}`); + } +} + +async function sha256Hex(input: string): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(input), + ); + let hex = ""; + for (const b of new Uint8Array(digest)) + hex += b.toString(16).padStart(2, "0"); + return hex; +} + +// Origin verification — the existing Lambda round-trip. Neon authoritative. +async function verifyOrigin( env: Env, bearerToken: string, -): Promise { +): Promise { const url = `${env.API_INTERNAL_URL.replace(/\/$/, "")}/internal/chat/verify-session`; const resp = await fetch(url, { method: "POST", @@ -31,7 +91,69 @@ export async function verifySession( if (resp.status !== 200) return null; - const body = (await resp.json()) as { userId?: string }; + const body = (await resp.json()) as { userId?: string; exp?: number }; if (!body.userId || typeof body.userId !== "string") return null; - return { userId: body.userId }; + // exp only bounds cache staleness — it must NOT gate auth. A missing/invalid + // exp (older origin during a rollout) yields null; auth still passes. + const exp = + typeof body.exp === "number" && Number.isFinite(body.exp) ? body.exp : null; + return { userId: body.userId, exp }; +} + +export async function verifySession( + env: Env, + bearerToken: string, +): Promise { + // Kill-switch: skip the cache entirely, verify at origin. + if (!cacheEnabled(env)) { + const origin = await verifyOrigin(env, bearerToken); + metric(env, "disabled"); + return origin ? { userId: origin.userId } : null; + } + + const key = await sha256Hex(bearerToken); + const nowSec = Math.floor(Date.now() / 1000); + + // Read-through. exp guard prevents serving a session that expired within the + // TTL window. KV failures are non-fatal — fall through to origin. + try { + const hit = await env.SESSION_CACHE.get(key, "json"); + if (hit && typeof hit.userId === "string" && hit.exp > nowSec) { + metric(env, "hit"); + return { userId: hit.userId }; + } + } catch { + // ignore — treat as a miss + } + + // Miss → origin. Negatives are NOT cached (ADR-0017 §1). + const origin = await verifyOrigin(env, bearerToken); + if (!origin) { + metric(env, "fail"); + return null; + } + + // Populate on success, but only with a real expiry to bound staleness. + // Without exp (older origin mid-rollout) we skip the write and degrade to + // no-cache rather than caching an unbounded entry. Best-effort — a write + // failure just means the next connect re-verifies at origin. + let wrote = false; + if (origin.exp !== null) { + try { + await env.SESSION_CACHE.put( + key, + JSON.stringify({ + userId: origin.userId, + exp: origin.exp, + } satisfies CachedSession), + { expirationTtl: cacheTtl(env) }, + ); + wrote = true; + } catch { + // non-fatal + } + } + + metric(env, "miss", wrote); + return { userId: origin.userId }; } diff --git a/services/chat-ws/worker-configuration.d.ts b/services/chat-ws/worker-configuration.d.ts index b89a2f4..064699a 100644 --- a/services/chat-ws/worker-configuration.d.ts +++ b/services/chat-ws/worker-configuration.d.ts @@ -20,6 +20,17 @@ declare global { // the chat-delivered topic; the chat-push Lambda (M6) dispatches APNs/FCM. CHAT_DELIVERED_TOPIC_ARN: string; AWS_REGION: string; + // Edge session cache (ADR-0017). Cache-aside store for WS-connect session + // verification, keyed by sha256(token). Raw namespace binding, not a + // linked secret. Read/write logic lands in B1.2+. + SESSION_CACHE: KVNamespace; + // TTL in seconds (string; parsed by the cache layer). Default 120. + SESSION_CACHE_TTL: string; + // Kill-switch: "0"/"false" → origin-only verification, no cache. + SESSION_CACHE_ENABLED: string; + // Load-test metrics (B1.7): "1" → emit one "SCM {...}" log per verify for + // wrangler-tail tally. Off ("0") in prod to avoid per-connect log volume. + SESSION_CACHE_METRICS: string; // Note: AWS creds for SigV4 publish come via `Resource.X.value` from // the SST `sst` import (linked CF Worker secrets pattern). They are NOT // exposed as env bindings; do not add them here. diff --git a/services/chat-ws/wrangler.jsonc b/services/chat-ws/wrangler.jsonc index 0342c25..9f341e0 100644 --- a/services/chat-ws/wrangler.jsonc +++ b/services/chat-ws/wrangler.jsonc @@ -16,6 +16,10 @@ { "name": "USER_INBOX", "class_name": "UserInbox" } ] }, + // Edge session cache (ADR-0017). Prod namespace_id is minted by SST + // (`sst.cloudflare.Kv` in infra/stacks/chat-ws.ts); this local id is a + // placeholder — `wrangler dev` simulates KV in-process and ignores it. + "kv_namespaces": [{ "binding": "SESSION_CACHE", "id": "session_cache_local" }], "migrations": [ { "tag": "v1", @@ -24,7 +28,11 @@ ], "vars": { // Override locally via .dev.vars for the dev API URL. - "API_INTERNAL_URL": "http://localhost:3001" + "API_INTERNAL_URL": "http://localhost:3001", + // Edge session cache config (mirrors SST env in chat-ws.ts). + "SESSION_CACHE_TTL": "120", + "SESSION_CACHE_ENABLED": "1", + "SESSION_CACHE_METRICS": "0" } // DATABASE_URL must be set in .dev.vars locally (not committed). SST // injects it in deployed stages; see infra/stacks/chat-ws.ts. From 9235fe0c5eeb696cc3fbdb9f786adb7eec33d4cb Mon Sep 17 00:00:00 2001 From: Alex Moreton Date: Thu, 2 Jul 2026 16:25:58 +0100 Subject: [PATCH 3/5] Invalidation + purge for cache eviction --- services/chat-ws/src/index.ts | 63 +++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/services/chat-ws/src/index.ts b/services/chat-ws/src/index.ts index f832da7..bbdde27 100644 --- a/services/chat-ws/src/index.ts +++ b/services/chat-ws/src/index.ts @@ -79,7 +79,7 @@ export default { }, } satisfies ExportedHandler; -// ── Internal push surface ─────────────────────────────────────────────────── +// ── Internal surface ──────────────────────────────────────────────────────── // // POST /internal/notify // header: x-chat-ws-secret: @@ -89,14 +89,31 @@ export default { // // API Lambda fans wake-up signals here (e.g. on mls.groups.publishWelcomes // success) so the recipient's UserInbox DOs can push to all connected -// sockets without waiting for the 30s background poll. Single trusted -// caller — the API Lambda — guarded by the same shared HMAC. +// sockets without waiting for the 30s background poll. +// +// POST /internal/session/purge (ADR-0017 §3 — write-through invalidation) +// header: x-chat-ws-secret: +// body: { tokenHash: string } sha256(token) hex, computed API-side +// 200: { purged: boolean } false = delete failed; TTL is the backstop +// 401: bad/missing secret +// +// The API's Better Auth sign-out / revoke hook (B1.4) computes sha256(token) +// and calls this so the edge session cache evicts immediately; the KV TTL +// bounds revocation lag if this call is ever missed. Raw tokens never reach +// this hop — only the hash — so a purge request can't be replayed as a bearer. +// +// Both endpoints share one trusted caller (the API Lambda) behind the same +// HMAC (verifyInternalAuth); never reachable from a client. interface InternalNotifyBody { userIds?: string[]; frame?: { kind?: string }; } +interface InternalPurgeBody { + tokenHash?: string; +} + function verifyInternalAuth(request: Request): boolean { const got = request.headers.get(INTERNAL_SECRET_HEADER); if (!got) return false; @@ -120,6 +137,10 @@ async function handleInternal( if (!verifyInternalAuth(request)) { return new Response("unauthorized", { status: 401 }); } + + if (pathname === "/internal/session/purge") { + return handleSessionPurge(request, env); + } if (pathname !== "/internal/notify") { return new Response("not found", { status: 404 }); } @@ -162,3 +183,39 @@ async function handleInternal( headers: { "content-type": "application/json" }, }); } + +// Evict a session from the edge cache (ADR-0017 §3). Body carries the hex +// sha256(token) — the same key auth.ts writes — so the raw token never reaches +// this hop. Best-effort: a failed delete returns { purged: false } (never 5xx), +// because the KV TTL is the revocation backstop and a caller must not fail its +// sign-out just because a cache eviction missed. +async function handleSessionPurge( + request: Request, + env: Env, +): Promise { + let body: InternalPurgeBody; + try { + body = (await request.json()) as InternalPurgeBody; + } catch { + return new Response("bad json", { status: 400 }); + } + + const hash = body.tokenHash; + // sha256 hex is exactly 64 lowercase hex chars — reject anything else so a + // malformed caller can't spray junk keys at KV. + if (typeof hash !== "string" || !/^[0-9a-f]{64}$/.test(hash)) { + return new Response("bad tokenHash", { status: 400 }); + } + + let purged = true; + try { + await env.SESSION_CACHE.delete(hash); + } catch { + purged = false; // TTL backstops the missed eviction + } + + return new Response(JSON.stringify({ purged }), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} From 3f50686d5c06827fd99bcd15685ff119109685bd Mon Sep 17 00:00:00 2001 From: Alex Moreton Date: Thu, 2 Jul 2026 16:28:24 +0100 Subject: [PATCH 4/5] Update v for latest dev deploy --- infra/stacks/chat-ws.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/stacks/chat-ws.ts b/infra/stacks/chat-ws.ts index d7ee392..ab89045 100644 --- a/infra/stacks/chat-ws.ts +++ b/infra/stacks/chat-ws.ts @@ -70,7 +70,7 @@ export const chatWsWorker = new sst.cloudflare.Worker("ChatWs", { // Load-test metrics (B1.7). "1" → per-verify "SCM" log for `wrangler tail` // to tally cache hit rate + KV write rate. Keep "0" in prod; flip to "1" // (+ redeploy) only for a load-test run, then flip back. - SESSION_CACHE_METRICS: "0", + SESSION_CACHE_METRICS: "1", }, transform: { worker: (args) => { @@ -142,8 +142,8 @@ export const chatWsWorker = new sst.cloudflare.Worker("ChatWs", { // is still v1 — in that case set { oldTag: "v1", newTag: "v2" } here and // let the KV binding ride that deploy, rather than jumping to v3. args.migrations = { - oldTag: "v2", - newTag: "v3", + oldTag: "v3", + newTag: "v4", }; // Compatibility date with WebSocket auto-reply-to-close behaviour From 11253b640e6f4d60f4c4dce302f3b9e70f8028c7 Mon Sep 17 00:00:00 2001 From: Alex Moreton Date: Thu, 2 Jul 2026 19:48:45 +0100 Subject: [PATCH 5/5] Commit cache miss/writes test using k6 --- loadtest/session-cache-connect.js | 183 ++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 loadtest/session-cache-connect.js diff --git a/loadtest/session-cache-connect.js b/loadtest/session-cache-connect.js new file mode 100644 index 0000000..13adf84 --- /dev/null +++ b/loadtest/session-cache-connect.js @@ -0,0 +1,183 @@ +// B1.7 — focused load test of the edge session cache (ADR-0017). +// +// Drives a WS connect / reconnect storm at the chat-ws Worker and measures +// connect latency. Pair it with the Worker's own counters: +// +// 1. Deploy chat-ws with SESSION_CACHE_METRICS=1 (infra/stacks/chat-ws.ts), +// then in another terminal: wrangler tail ChatWs --format=json | grep SCM +// 2. Run this script. Tally the SCM lines: r="hit"/"miss" = hit rate, +// w=true = KV write rate (ADR-0017 §5 — the cost metric to watch). +// +// Compare two runs — cache ON vs OFF (flip SESSION_CACHE_ENABLED, redeploy): +// the delta in sc_connect_latency p95 is the DB-hot-path saving the cache buys. +// +// Tokens: setup() self-provisions LT_USERS test accounts (sign-up, ignore if +// they exist) and signs them in for real bearer tokens. VUs reuse tokens, so the +// first connect per token is a cache miss (populate) and the rest are hits. +// +// Usage: +// k6 run \ +// -e API_URL=https:// \ +// -e WS_URL=wss:// \ +// -e LT_USERS=200 -e LT_PASSWORD=loadtest-pw-123 \ +// loadtest/session-cache-connect.js + +import ws from "k6/ws"; +import http from "k6/http"; +import encoding from "k6/encoding"; +import { check, sleep } from "k6"; +import { Trend, Counter } from "k6/metrics"; + +const API_URL = (__ENV.API_URL || "http://localhost:3001").replace(/\/$/, ""); +const WS_URL = (__ENV.WS_URL || "ws://localhost:8787").replace(/\/$/, ""); +const N_USERS = parseInt(__ENV.LT_USERS || "200", 10); +const PASSWORD = __ENV.LT_PASSWORD || "loadtest-pw-123"; +const EMAIL_PREFIX = __ENV.LT_EMAIL_PREFIX || "loadtest"; +// Run-scoped so every run provisions brand-new users → all sign-up (1 request, +// token via autoSignIn), deterministic 200/200, no cross-run password/sign-in +// dependency. Pass the same LT_RUN_ID to two runs to reuse a pool (ON vs OFF). +const RUN_ID = __ENV.LT_RUN_ID || String(Date.now()); +const HOLD_MS = parseInt(__ENV.LT_HOLD_MS || "500", 10); +// Seconds to wait between setup() auth calls. Raise (e.g. 0.2) if Better Auth +// rate-limits the provisioning burst (429). Default 0 = as fast as possible. +const SETUP_DELAY = parseFloat(__ENV.LT_SETUP_DELAY || "0"); + +// Better Auth rejects auth requests with no Origin header (CSRF: +// MISSING_OR_NULL_ORIGIN). It auto-trusts its own baseURL, so sending the API's +// own origin passes. Override with LT_ORIGIN if your TRUSTED_ORIGINS differs. +function originOf(u) { + const m = /^(https?:\/\/[^/]+)/.exec(u); + return m ? m[1] : u; +} +const ORIGIN = __ENV.LT_ORIGIN || originOf(API_URL); +const AUTH_HEADERS = { "content-type": "application/json", Origin: ORIGIN }; + +const connectLatency = new Trend("sc_connect_latency", true); +const connectOk = new Counter("sc_connect_ok"); +const connectFail = new Counter("sc_connect_fail"); + +export const options = { + // Provisioning is sequential auth calls (~0.5s each on Lambda+Neon), so the + // 60s default is too tight for a large pool. Raise it; drop LT_USERS for speed. + setupTimeout: __ENV.LT_SETUP_TIMEOUT || "300s", + scenarios: { + // Reconnect storm — each iteration is one connect+close, so ramping VUs and + // looping default() reproduces app-foreground / network-flap reconnects. + reconnect_storm: { + executor: "ramping-vus", + startVUs: 0, + stages: [ + { duration: "30s", target: 100 }, + { duration: "1m", target: 500 }, + { duration: "30s", target: 0 }, + ], + gracefulStop: "10s", + }, + }, + thresholds: { + // Tune against your baseline; the point is the ON-vs-OFF delta, not absolutes. + sc_connect_latency: ["p(95)<1500"], + sc_connect_fail: ["count<50"], + }, +}; + +// Better Auth returns the session token in the `set-auth-token` response header +// (bearer plugin). k6 may title-case header keys, so read case-insensitively. +function readHeader(headers, name) { + const want = name.toLowerCase(); + for (const k in headers) { + if (k.toLowerCase() === want) return headers[k]; + } + return null; +} + +function signUp(email) { + // 200 = created; 4xx (e.g. already exists) is fine — we sign in next. + return http.post( + `${API_URL}/auth/sign-up/email`, + JSON.stringify({ email, password: PASSWORD, name: email }), + { headers: AUTH_HEADERS }, + ); +} + +function signIn(email) { + return http.post( + `${API_URL}/auth/sign-in/email`, + JSON.stringify({ email, password: PASSWORD }), + { headers: AUTH_HEADERS }, + ); +} + +export function setup() { + const tokens = []; + const via = { signin: 0, signup: 0, fail: 0 }; + let sampled = 0; + for (let i = 0; i < N_USERS; i++) { + const email = `${EMAIL_PREFIX}+${RUN_ID}-${i}@example.com`; + // Sign-up first: a brand-new user (unique per run) mints in ONE request via + // autoSignIn (set-auth-token header). Sign-in is only a fallback for the + // reused-LT_RUN_ID case (user already exists → 422). + let res = signUp(email); + let token = readHeader(res.headers, "set-auth-token"); + if (token) { + via.signup++; + } else { + res = signIn(email); + token = readHeader(res.headers, "set-auth-token"); + if (token) via.signin++; + } + if (token) { + tokens.push(token); + } else { + via.fail++; + if (sampled < 3) { + sampled++; + console.error( + `setup: ${email} no token; last ${res.status}: ${String(res.body).slice(0, 160)}`, + ); + } + } + if (SETUP_DELAY > 0) sleep(SETUP_DELAY); + } + console.info( + `setup: minted ${tokens.length}/${N_USERS} tokens ` + + `(signin=${via.signin} signup=${via.signup} fail=${via.fail})`, + ); + if (tokens.length === 0) { + throw new Error( + "setup: minted 0 tokens — check API_URL / Origin / credentials", + ); + } + return { tokens }; +} + +export default function (data) { + const { tokens } = data; + // Reuse tokens across VUs → mostly cache hits after the first populate. + const token = tokens[(__VU + __ITER) % tokens.length]; + // base64url (no padding) — mirrors packages/chat-transport/src/client.ts, which + // the Worker decodes in index.ts (Sec-WebSocket-Protocol tchar-safe). + const encoded = encoding.b64encode(token, "rawurl"); + + const params = { + headers: { + // "bearer, " — exactly what the real client sends. + "Sec-WebSocket-Protocol": `bearer, ${encoded}`, + }, + }; + + const start = Date.now(); + const res = ws.connect(WS_URL, params, function (socket) { + socket.on("open", function () { + connectLatency.add(Date.now() - start); + connectOk.add(1); + // Hold, then close so the next iteration reconnects. + socket.setTimeout(function () { + socket.close(); + }, HOLD_MS); + }); + }); + + const ok = check(res, { "ws handshake 101": (r) => r && r.status === 101 }); + if (!ok) connectFail.add(1); +}