diff --git a/.github/workflows/cloudflare-pages.yml b/.github/workflows/cloudflare-pages.yml index 3225720..a0812ae 100644 --- a/.github/workflows/cloudflare-pages.yml +++ b/.github/workflows/cloudflare-pages.yml @@ -173,8 +173,13 @@ jobs: # and deployed as is (--no-bundle), so production runs the tested bundle. No token in this job. run: | npm ci --prefix deploy + # site.request's release blob (telemetry.v1): the bundle names the commit it was built from. The committed + # server/release.js says unknown, which is what tests and local runs record. + printf "export const RELEASE = '%s';\n" "${GITHUB_SHA::7}" > server/release.js + grep -Eq "^export const RELEASE = '[0-9a-f]{7}';$" server/release.js deploy/node_modules/.bin/wrangler pages functions build --outdir dist/_worker.js --compatibility-date 2026-10-01 test -s dist/_worker.js/index.js + grep -Fq "RELEASE = \"${GITHUB_SHA::7}\"" dist/_worker.js/index.js grep -Fq '"/admin/*"' dist/_routes.json if grep -Fq '"/*"' dist/_routes.json; then echo 'routes too wide'; exit 1; fi if grep -rqF 'GOCSPX-' dist; then echo 'Google client secret in dist'; exit 1; fi diff --git a/server/CONTRACT.md b/server/CONTRACT.md index 120adc1..a4060ad 100644 --- a/server/CONTRACT.md +++ b/server/CONTRACT.md @@ -142,6 +142,7 @@ and `helpers.js` (`makeEnv`, `seedRole`, `seedUser`, `seedSession`, `signIn`, `c | views.test.js | views (Packet 2) | 22, escaping of names, nav filtered by perms | | changes.test.js | Packet 3 | 3, 12 (code + escaping) | | people.test.js, roles.test.js, guards.test.js | Packet 4 | 13, 14, 19, 20, 21, 23 | +| telemetry.test.js | site.request (RFC 0004) | telemetry.v1 conformance | ## Contract changes @@ -341,3 +342,19 @@ introduced by this seam. App local stores remain authoritative; uploads and vali explicit user actions. Tests: account-api.test.js includes owner isolation, CORS/CSRF, quotas, retries, input bounds, disabled/expired sessions, account-switch races in both directions and fail-closed storage errors. + +## site.request counts (telemetry.v1, 2026-10-31) + +MasterMind RFC 0004 and `docs/contracts/telemetry-v1.md`. `handle()` writes one Analytics Engine data point per +response through `env.EVENTS`, after the response is final, inside a try and never awaited: `indexes +['site.request']`, `blobs [event, release, route, method, status, code, app]`, `doubles [1, ms]`. Only the +production configuration declares the binding (`wrangler.toml`; preview declares none). `telemetry/events.json` +is the allowlist: **telemetry.js** `sitePoint(fields)` builds the point from it or returns null, and +`tag(res, code)` / `codeOf(res)` keep an error code beside a Response (`fail()`, the 403 page, `unavailable()`, +the account API's `failure()`), read before `secure()` copies it. `route` is `routes.template(table, pathname)` +or `account-api.accountRoute(pathname)`, else `unmatched`; `app` is the request Origin against `APP_ORIGINS` +and `ORIGIN`, else `none`; `release` is **release.js** `RELEASE`, `unknown` as committed and the short sha in +the bundle CI builds. No user, session or save id, address, User-Agent, raw path or query string is written. +Tests: telemetry.test.js holds responses byte-identical with the binding absent, present, throwing and +malformed, checks every point against the allowlist and its closed sets, and pushes input text into every +request field. diff --git a/server/account-api.js b/server/account-api.js index fadda24..2f43fc5 100644 --- a/server/account-api.js +++ b/server/account-api.js @@ -2,15 +2,23 @@ import { loadSession } from './session.js'; import { newId } from './crypto.js'; import { identityName } from './authz.js'; +import { tag } from './telemetry.js'; export const ACCOUNT_BASE = '/admin/api/v1'; export const APP_ORIGINS = { omni: 'https://omni.syberlabs.io', rise: 'https://rise.syberlabs.io', sketch: 'https://sketch.syberlabs.io' }; export const isAccountApi = pathname => pathname === ACCOUNT_BASE || pathname.startsWith(ACCOUNT_BASE + '/'); +// site.request's route for an API path: one of these three templates, never the path, which can carry a save id. +export function accountRoute(pathname) { + const rest = isAccountApi(pathname) ? pathname.slice(ACCOUNT_BASE.length) : null; + if (rest === '/account' || rest === '/saves') return ACCOUNT_BASE + rest; + return /^\/saves\/[^/]+$/.test(rest) ? `${ACCOUNT_BASE}/saves/:id` : null; +} const MAX_BYTES = 1048576; const ID = /^[a-zA-Z0-9_-]{8,80}$/; const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const json = (data, status = 200) => Response.json({ version: 1, ...data }, { status }); -const failure = (error, status) => json({ error }, status); +// The error is also kept beside the response for site.request's code (telemetry.js tag). +const failure = (error, status) => tag(json({ error }, status), error); const metadata = row => ({ id: row.id, app: row.app, name: row.name, createdAt: row.created_at, bytes: row.bytes }); export function accountCors(res, request, env) { diff --git a/server/app.js b/server/app.js index 4cad893..bd9f870 100644 --- a/server/app.js +++ b/server/app.js @@ -1,12 +1,15 @@ // The Function's front door. Order matters: ORIGIN -> canonical host -> config -> method rules -> /admin -// -> CSRF -> session -> route -> access -> form -> handler. Every response leaves through secure(). +// -> CSRF -> session -> route -> access -> form -> handler. Every response leaves through secure(), and is +// then counted once as a site.request data point (telemetry.v1) that cannot change it. import { CSP, CSP_WITH_GOOGLE, HttpError, checkCsrf, clearCookie, html, readForm, redirect, secure } from './http.js'; import { secretKeyBytes } from './crypto.js'; import { PUBLIC, SIGNED_IN } from './permissions.js'; -import { TABLE, match, ownerPage, tableFor } from './routes.js'; +import { TABLE, match, ownerPage, tableFor, template } from './routes.js'; import { googleEnabled } from './providers.js'; import { SESSION_COOKIE, loadSession } from './session.js'; -import { accountApi, accountCors, isAccountApi } from './account-api.js'; +import { APP_ORIGINS, accountApi, accountCors, accountRoute, isAccountApi } from './account-api.js'; +import { codeOf, sitePoint, tag } from './telemetry.js'; +import { RELEASE } from './release.js'; import { errorHtml, forbiddenHtml, notFoundHtml, unavailableHtml } from './views/forbidden.js'; const REQUIRED = ['GITHUB_CLIENT_ID', 'GITHUB_CLIENT_SECRET', 'APP_SECRET']; @@ -17,14 +20,18 @@ const configured = env => REQUIRED.every(k => typeof env[k] === 'string' && env[ const isAdminPath = p => p === '/admin' || p.startsWith('/admin/'); const noHead = p => p.startsWith('/auth/start/') || p.startsWith('/auth/callback/'); -// RFC-0002 2.1, 3.4: rendered in place on the path that failed, so a reload retries. -const unavailable = pathname => html(unavailableHtml(pathname), { status: 503 }); +// RFC-0002 2.1, 3.4: rendered in place on the path that failed, so a reload retries. The account API answers +// the same failure in its own JSON shape. Each keeps its code beside it for site.request (telemetry.js tag). +const unavailable = pathname => tag(html(unavailableHtml(pathname), { status: 503 }), 'unavailable'); +const down = pathname => (isAccountApi(pathname) + ? tag(Response.json({ version: 1, error: 'unavailable' }, { status: 503 }), 'unavailable') + : unavailable(pathname)); // Every error is an HTML page (RFC-0002 R1-8: no /api/*, so no JSON error shape). function fail(ctx, code) { const err = code instanceof HttpError ? code : new HttpError(code); - if (err.code === 'not_found') return html(notFoundHtml(ctx), { status: 404 }); - return html(errorHtml(ctx, err), { status: err.status }); + if (err.code === 'not_found') return tag(html(notFoundHtml(ctx), { status: 404 }), 'not_found'); + return tag(html(errorHtml(ctx, err), { status: err.status }), err.code); } function methodNotAllowed(ctx, allow) { @@ -72,7 +79,7 @@ async function route(request, env, now) { const { access } = found.route; if (access !== PUBLIC) { if (!ctx.user) return signinRequired(ctx, session.stale); - if (access !== SIGNED_IN && !ctx.perms.has(access)) return html(forbiddenHtml(ctx, access), { status: 403 }); + if (access !== SIGNED_IN && !ctx.perms.has(access)) return tag(html(forbiddenHtml(ctx, access), { status: 403 }), 'forbidden'); } try { if (method === 'POST') ctx.form = await readForm(request); @@ -84,10 +91,11 @@ async function route(request, env, now) { } export async function handle(request, env, now = Date.now()) { + const started = Date.now(); const url = new URL(request.url); let res; if (!env || typeof env.ORIGIN !== 'string' || !env.ORIGIN) { - res = isAccountApi(url.pathname) ? Response.json({ version: 1, error: 'unavailable' }, { status: 503 }) : unavailable(url.pathname); + res = down(url.pathname); } else if (url.origin !== env.ORIGIN) { // pages.dev and preview hosts never serve the app; same path on the canonical origin. res = redirect(env.ORIGIN + url.pathname + url.search, { status: 308 }); @@ -97,11 +105,46 @@ export async function handle(request, env, now = Date.now()) { } catch (err) { // Fail closed (D1 down, a bug). The message may hold data, so only its class is logged. console.error(`staff: unhandled ${err && err.name ? err.name : 'error'}`); - res = isAccountApi(url.pathname) ? Response.json({ version: 1, error: 'unavailable' }, { status: 503 }) : unavailable(url.pathname); + res = down(url.pathname); } } + const code = codeOf(res); res = secure(res, googleEnabled(env) ? CSP_WITH_GOOGLE : CSP); if (isAccountApi(url.pathname) && url.origin === env?.ORIGIN) res = accountCors(res, request, env); - if (request.method === 'HEAD') return new Response(null, { status: res.status, statusText: res.statusText, headers: res.headers }); + if (request.method === 'HEAD') res = new Response(null, { status: res.status, statusText: res.statusText, headers: res.headers }); + count(request, env, url, res, code, started); return res; } + +const METHODS = ['GET', 'HEAD', 'POST', 'OPTIONS']; + +// The request's Origin against the account API's allowlist. Browsers send no Origin on a same-origin GET (the +// site's own account check among them), so `site` counts the site's own form posts, and `none` the rest. +function appOf(request, env) { + const origin = request.headers.get('Origin'); + if (origin && origin === env.ORIGIN) return 'site'; + return Object.keys(APP_ORIGINS).find(app => APP_ORIGINS[app] === origin) || 'none'; +} + +// One site.request data point per response, written once the response is final (telemetry.v1 producer rules +// 1 and 2): never awaited and inside a try, so an absent, broken or throwing binding cannot change a byte of it. +// Only production declares the EVENTS binding (wrangler.toml): preview deployments skip all of this, and +// wrangler's local binding discards what it is given. +function count(request, env, url, res, code, started) { + if (!env?.EVENTS) return; + try { + const route = (isAccountApi(url.pathname) ? accountRoute(url.pathname) : template(tableFor(env), url.pathname)) || 'unmatched'; + const point = sitePoint({ + release: RELEASE, + route, + method: METHODS.includes(request.method) ? request.method : 'other', + status: `${Math.floor(res.status / 100)}xx`, + code, + app: appOf(request, env), + ms: Math.max(0, Date.now() - started), + }); + if (point) env.EVENTS.writeDataPoint(point); + } catch { + // A failed write costs one data point, never a response. + } +} diff --git a/server/release.js b/server/release.js new file mode 100644 index 0000000..26b85e4 --- /dev/null +++ b/server/release.js @@ -0,0 +1,3 @@ +// The deployed commit, for site.request's release blob (telemetry.v1). CI's Function build rewrites this file +// with the short sha just before it bundles (.github/workflows/cloudflare-pages.yml); anywhere else it is unknown. +export const RELEASE = 'unknown'; diff --git a/server/routes.js b/server/routes.js index 079abbb..fb8b554 100644 --- a/server/routes.js +++ b/server/routes.js @@ -105,6 +105,13 @@ export function match(table, method, pathname) { return methods.size ? { methods: [...methods].sort() } : null; } +// The pattern a path matches, whatever the method, or null. site.request records this template, never the +// raw path, which can carry an id. +export function template(table, pathname) { + const parts = pathname.split('/'); + return table.find(route => matchPath(route.segments, parts))?.pattern || null; +} + // For a signed-out POST: the longest parameterless GET route that owns the form's path (RFC-0002 3.6). export function ownerPage(table, pathname) { let best = '/admin/'; diff --git a/server/telemetry.js b/server/telemetry.js new file mode 100644 index 0000000..6352d9e --- /dev/null +++ b/server/telemetry.js @@ -0,0 +1,41 @@ +// telemetry.v1 (MasterMind docs/contracts/telemetry-v1.md, RFC 0004): the site.request data point, built only +// from what telemetry/events.json lists. A field the allowlist does not name is never written, and a value +// outside a field's list drops the whole point. Nothing here sees a user, session or save id, an address, a +// User-Agent, a raw path or a query string: app.js passes a route template, a status class and its own +// constants. No imports but the allowlist, so app.js and account-api.js can both use it without an import cycle. +import events from '../telemetry/events.json' with { type: 'json' }; + +export const EVENT = 'site.request'; +const SPEC = events.events[EVENT]; + +// The error code a response was built with (an HttpError's code, the account API's error), kept beside the +// Response rather than read back from its body. app.js reads it before secure() copies the response. +const codes = new WeakMap(); +export const tag = (res, code) => { codes.set(res, code); return res; }; +export const codeOf = res => codes.get(res) || 'none'; + +// release, route and code have no values list: each comes from a closed set the code owns (the short sha CI +// writes into release.js, a route-table template, an error-code literal). These checks refuse any shape no +// member of those sets has, and a blob added to the allowlist without a check here writes nothing. +const CLOSED = { + release: value => /^(?:[0-9a-f]{7}|unknown)$/.test(value), + route: value => value === 'unmatched' || /^\/[a-z0-9/:-]*$/.test(value), + code: value => /^[a-z][a-z_]{0,39}$/.test(value), +}; + +// { indexes: [event], blobs: [event, ...the allowlist's blobs in order], doubles: [1, ...its numbers] }, or null. +export function sitePoint(fields) { + const blobs = [EVENT]; + for (const { name, values } of SPEC.blobs) { + const value = fields[name]; + if (typeof value !== 'string' || !(values ? values.includes(value) : CLOSED[name]?.(value))) return null; + blobs.push(value); + } + const doubles = []; + for (const name of SPEC.doubles) { + const value = name === 'count' ? 1 : fields[name]; + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) return null; + doubles.push(value); + } + return { indexes: [EVENT], blobs, doubles }; +} diff --git a/server/test/telemetry.test.js b/server/test/telemetry.test.js new file mode 100644 index 0000000..ed1abb4 --- /dev/null +++ b/server/test/telemetry.test.js @@ -0,0 +1,269 @@ +// telemetry.v1 conformance for site.request (MasterMind docs/contracts/telemetry-v1.md, RFC 0004): one data point +// per response from handle(), only the allowlist's fields and values, route templates instead of paths, no input +// text in any field, responses byte-identical with the binding absent, present and throwing, and the binding in +// the production configuration only. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { randomUUID } from 'node:crypto'; +import { handle } from '../app.js'; +import { TABLE } from '../routes.js'; +import { ERRORS } from '../http.js'; +import { sitePoint } from '../telemetry.js'; +import { RELEASE } from '../release.js'; +import { brokenDb } from './d1-shim.js'; +import { ORIGIN, SESSION_COOKIE, call, makeEnv, signIn } from './helpers.js'; + +const read = path => readFile(new URL(path, import.meta.url), 'utf8'); +const ALLOWLIST = JSON.parse(await read('../../telemetry/events.json')); +const SPEC = ALLOWLIST.events['site.request']; +const API = '/admin/api/v1'; +const OMNI = 'https://omni.syberlabs.io'; +const RISE = 'https://rise.syberlabs.io'; +const SKETCH = 'https://sketch.syberlabs.io'; +const NOW = Date.now(); + +// The closed sets the free blobs must come from: the route table's templates and the account API's three, and the +// error codes the code itself names (ERRORS, the app's own, and every literal the account API fails with). +const ACCOUNT_SOURCE = await read('../account-api.js'); +const TEMPLATES = new Set([...TABLE.map(r => r.pattern), `${API}/account`, `${API}/saves`, `${API}/saves/:id`, 'unmatched']); +const CODES = new Set([...Object.keys(ERRORS), 'forbidden', 'unavailable', 'none', + ...[...ACCOUNT_SOURCE.matchAll(/failure\('([a-z_]+)'|code: '([a-z_]+)'/g)].map(m => m[1] || m[2])]); + +const recorder = () => { + const points = []; + return { points, writeDataPoint: point => { points.push(point); } }; +}; + +// helpers.call, plus a raw body for the account API's JSON save. +async function send(env, path, opts = {}) { + if (opts.body === undefined) return call(env, path, opts); + const headers = new Headers(opts.headers || {}); + if (opts.cookie) headers.set('Cookie', opts.cookie); + if (opts.origin) headers.set('Origin', opts.origin); + return handle(new Request(new URL(path, ORIGIN), { method: opts.method || 'POST', headers, body: opts.body }), env, opts.now); +} +const field = (point, name) => point.blobs[1 + SPEC.blobs.findIndex(b => b.name === name)]; +const fields = point => Object.fromEntries(SPEC.blobs.map(b => [b.name, field(point, b.name)])); + +// An admin with one omni backup, a viewer, and the requests every test below replays: pages, redirects, refusals +// and the account API from each consumer origin. +async function fixture() { + const env = makeEnv(); + const admin = await signIn(env.DB, { roles: ['role_admin'], now: NOW }); + const viewer = await signIn(env.DB, { roles: ['role_viewer'], now: NOW }); + const saveId = 'save-fixture-0123456789'; + env.DB.sqlite.prepare('INSERT INTO account_saves (id, user_id, app, name, payload, bytes, request_id, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + .run(saveId, admin.userId, 'omni', 'Private workspace', '{"documents":["hello"]}', 23, randomUUID(), NOW); + const expect = { 'X-SyberLabs-Expected-User': admin.userId }; + const requests = [ + ['/auth/signin', {}], + ['/auth/signin?e=expired', {}], + ['/auth/callback/github?code=c&state=s', {}], + ['/admin/', {}], + ['/admin/', { cookie: admin.cookie }], + ['/admin/', { cookie: admin.cookie, method: 'HEAD' }], + ['/admin', {}], + ['/admin/people', { cookie: admin.cookie }], + ['/admin/people', { cookie: viewer.cookie }], + ['/admin/nope', { cookie: admin.cookie }], + ['/admin/changes', { cookie: admin.cookie, method: 'PUT' }], + ['/admin/people/add', { cookie: admin.cookie }], + ['/auth/start/github', { method: 'HEAD' }], + ['/admin/people/add', { cookie: admin.cookie, form: { account: 'octocat' }, origin: 'https://evil.example' }], + ['/auth/signout', { form: {} }], + [`/admin/saves/${saveId}/download`, { cookie: admin.cookie }], + ['/admin/saves/missing-save-id/download', { cookie: admin.cookie }], + ['/admin/changes?x=1', { host: 'https://syberlabs-home.pages.dev' }], + [`${API}/account`, { headers: { Origin: RISE } }], + [`${API}/account`, { cookie: admin.cookie, headers: { Origin: OMNI } }], + [`${API}/account`, { cookie: admin.cookie }], + [`${API}/saves`, { method: 'OPTIONS', origin: SKETCH, headers: { + 'Access-Control-Request-Method': 'POST', 'Access-Control-Request-Headers': 'content-type, x-syberlabs-account' } }], + [`${API}/saves?app=omni`, { cookie: admin.cookie, headers: { ...expect, Origin: OMNI } }], + [`${API}/saves/${saveId}`, { cookie: admin.cookie, headers: { ...expect, Origin: OMNI } }], + [`${API}/saves/${saveId}`, { cookie: admin.cookie, headers: { ...expect, Origin: RISE } }], + [`${API}/saves`, { cookie: admin.cookie, method: 'POST', origin: 'https://evil.example', headers: expect }], + [`${API}/nope/deeper`, { cookie: admin.cookie }], + ]; + return { env, admin, viewer, saveId, requests }; +} + +async function snapshot(res) { + return { status: res.status, statusText: res.statusText, headers: [...res.headers], body: await res.text() }; +} + +test('responses are byte-identical with the EVENTS binding absent, present, throwing or malformed', async () => { + const { env, requests } = await fixture(); + const recording = recorder(); + const variants = [ + { ...env }, + { ...env, EVENTS: recording }, + { ...env, EVENTS: { writeDataPoint() { throw new Error('analytics engine down'); } } }, + { ...env, EVENTS: {} }, + ]; + for (const [path, opts] of requests) { + const seen = []; + for (const variant of variants) seen.push(await snapshot(await call(variant, path, { ...opts, now: NOW }))); + for (let i = 1; i < seen.length; i++) assert.deepEqual(seen[i], seen[0], `${opts.method || 'GET'} ${path}, variant ${i}`); + } + assert.equal(recording.points.length, requests.length, 'one data point per response'); + + // The failure paths too: D1 down and missing config, on a page and on the account API. + const realError = console.error; + console.error = () => {}; + try { + const { cookie } = await signIn(makeEnv().DB, { roles: ['role_admin'] }); + for (const base of [makeEnv({ DB: brokenDb() }), makeEnv({ APP_SECRET: undefined })]) { + for (const path of ['/admin/', `${API}/account`]) { + const seen = []; + for (const EVENTS of [undefined, recorder(), { writeDataPoint() { throw new Error('down'); } }]) { + seen.push(await snapshot(await call({ ...base, EVENTS }, path, { cookie, now: NOW }))); + } + assert.equal(seen[0].status, 503, path); + for (let i = 1; i < seen.length; i++) assert.deepEqual(seen[i], seen[0], `${path}, variant ${i}`); + } + } + } finally { + console.error = realError; + } +}); + +test('every data point holds only the allowlist: its fields, in order, from their closed sets', async () => { + const { env, requests } = await fixture(); + const recording = recorder(); + env.EVENTS = recording; + for (const [path, opts] of requests) await call(env, path, { ...opts, now: NOW }); + assert.equal(recording.points.length, requests.length); + for (const point of recording.points) { + const label = JSON.stringify(point); + assert.deepEqual(Object.keys(point).sort(), ['blobs', 'doubles', 'indexes'], label); + assert.deepEqual(point.indexes, ['site.request'], label); + assert.equal(point.blobs.length, 1 + SPEC.blobs.length, label); + assert.equal(point.blobs[0], 'site.request', label); + for (const { name, values } of SPEC.blobs) if (values) assert.ok(values.includes(field(point, name)), `${name}: ${label}`); + assert.equal(field(point, 'release'), RELEASE, label); + assert.ok(TEMPLATES.has(field(point, 'route')), `route: ${label}`); + assert.ok(CODES.has(field(point, 'code')), `code: ${label}`); + assert.equal(point.doubles.length, SPEC.doubles.length, label); + assert.equal(point.doubles[0], 1, label); + assert.ok(Number.isFinite(point.doubles[1]) && point.doubles[1] >= 0, label); + } + assert.equal(RELEASE, 'unknown', 'the committed release.js; CI writes the short sha before the Function build'); +}); + +test('routes are recorded as templates, with method, status class, error code and consumer app', async () => { + const { env, admin, viewer, saveId } = await fixture(); + const recording = recorder(); + env.EVENTS = recording; + const one = async (path, opts = {}) => { + await send(env, path, { now: NOW, ...opts }); + return fields(recording.points.at(-1)); + }; + const at = (route, method, status, code, app) => ({ release: RELEASE, route, method, status, code, app }); + const expect = { 'X-SyberLabs-Expected-User': admin.userId }; + + assert.deepEqual(await one(`/admin/saves/${saveId}/download`, { cookie: admin.cookie }), at('/admin/saves/:id/download', 'GET', '2xx', 'none', 'none')); + assert.deepEqual(await one(`${API}/saves/${saveId}`, { cookie: admin.cookie, headers: { ...expect, Origin: OMNI } }), at(`${API}/saves/:id`, 'GET', '2xx', 'none', 'omni')); + assert.deepEqual(await one(`${API}/saves/${saveId}`, { cookie: admin.cookie, headers: { ...expect, Origin: RISE } }), at(`${API}/saves/:id`, 'GET', '4xx', 'not_found', 'rise')); + assert.deepEqual(await one('/auth/callback/github?code=c&state=s'), at('/auth/callback/:provider', 'GET', '3xx', 'none', 'none')); + assert.deepEqual(await one(`${API}/account`, { headers: { Origin: RISE } }), at(`${API}/account`, 'GET', '4xx', 'signin_required', 'rise')); + assert.deepEqual(await one(`${API}/saves?app=sketch`, { cookie: admin.cookie, headers: { Origin: SKETCH } }), at(`${API}/saves`, 'GET', '4xx', 'expected_user_required', 'sketch')); + assert.deepEqual(await one(`${API}/saves`, { method: 'OPTIONS', origin: SKETCH, headers: { 'Access-Control-Request-Method': 'POST' } }), at(`${API}/saves`, 'OPTIONS', '2xx', 'none', 'sketch')); + assert.deepEqual(await one(`${API}/saves`, { cookie: admin.cookie, method: 'POST', origin: OMNI, headers: { ...expect, 'Content-Type': 'application/json', 'X-SyberLabs-Account': 'v1' }, + body: JSON.stringify({ app: 'omni', name: 'n', payload: {}, requestId: randomUUID() }) }), at(`${API}/saves`, 'POST', '2xx', 'none', 'omni')); + assert.deepEqual(await one(`${API}/nope/deeper`, { cookie: admin.cookie }), at('unmatched', 'GET', '4xx', 'not_found', 'none')); + assert.deepEqual(await one('/admin/people/add', { cookie: admin.cookie, form: { account: 'octocat' }, origin: 'https://evil.example' }), at('/admin/people/add', 'POST', '4xx', 'csrf', 'none')); + assert.deepEqual(await one('/admin/changes', { cookie: admin.cookie, form: {} }), at('/admin/changes', 'POST', '4xx', 'none', 'site')); + assert.deepEqual(await one('/admin/changes', { cookie: admin.cookie, method: 'PUT' }), at('/admin/changes', 'other', '4xx', 'method_not_allowed', 'site')); + assert.deepEqual(await one('/auth/start/github', { method: 'HEAD' }), at('/auth/start/:provider', 'HEAD', '4xx', 'method_not_allowed', 'none')); + assert.deepEqual(await one('/admin/nope', { cookie: admin.cookie }), at('unmatched', 'GET', '4xx', 'not_found', 'none')); + assert.deepEqual(await one('/admin/people', { cookie: viewer.cookie }), at('/admin/people', 'GET', '4xx', 'forbidden', 'none')); + assert.deepEqual(await one('/admin/changes?x=1', { host: 'https://syberlabs-home.pages.dev' }), at('/admin/changes', 'GET', '3xx', 'none', 'none')); + assert.deepEqual(await one('/admin/account/add-google', { cookie: admin.cookie }), at('unmatched', 'GET', '4xx', 'not_found', 'none'), 'Google off: not in the table'); + assert.deepEqual(await one('/admin/changes', { cookie: admin.cookie, form: { x: 'y'.repeat(20000) } }), at('/admin/changes', 'POST', '4xx', 'too_large', 'site')); + + const realError = console.error; + console.error = () => {}; + try { + const broken = { ...makeEnv({ DB: brokenDb() }), EVENTS: recording }; + await call(broken, '/admin/', { cookie: admin.cookie }); + assert.deepEqual(fields(recording.points.at(-1)), at('/admin/', 'GET', '5xx', 'unavailable', 'none')); + await call(broken, `${API}/account`, { cookie: admin.cookie, headers: { Origin: OMNI } }); + assert.deepEqual(fields(recording.points.at(-1)), at(`${API}/account`, 'GET', '5xx', 'unavailable', 'omni')); + } finally { + console.error = realError; + } +}); + +test('text pushed into every request field never reaches a data point', async () => { + const { env, admin, saveId } = await fixture(); + const recording = recorder(); + env.EVENTS = recording; + const canary = 'canary-7f3e'; + const tries = [ + [`/admin/saves/${canary}-save/download`, { cookie: admin.cookie }], + [`/admin/changes?q=${canary}&e=${canary}`, { cookie: admin.cookie }], + [`${API}/saves/${canary}-0123`, { cookie: admin.cookie, headers: { 'X-SyberLabs-Expected-User': admin.userId } }], + [`${API}/saves?app=${canary}`, { cookie: admin.cookie, headers: { 'X-SyberLabs-Expected-User': `${canary}-user` } }], + [`/auth/${canary}`, { headers: { 'User-Agent': `Mozilla/5.0 ${canary}`, 'CF-Connecting-IP': '203.0.113.77' } }], + [`/admin/${canary}`, { cookie: `${SESSION_COOKIE}=${canary}${'A'.repeat(32)}`, headers: { Origin: `https://${canary}.example` } }], + [`/auth/callback/${canary}?code=${canary}&state=${canary}`, {}], + ['/admin/changes', { cookie: admin.cookie, method: 'PROPFIND', headers: { Referer: `${ORIGIN}/${canary}` } }], + [`${API}/saves`, { cookie: admin.cookie, method: 'POST', origin: OMNI, headers: { 'X-SyberLabs-Expected-User': admin.userId, 'Content-Type': 'application/json', 'X-SyberLabs-Account': 'v1' }, + body: JSON.stringify({ app: 'omni', name: canary, payload: { text: canary }, requestId: randomUUID() }) }], + [`/admin/saves/${saveId}/download`, { cookie: admin.cookie }], + ]; + for (const [path, opts] of tries) await send(env, path, { now: NOW, ...opts }); + assert.equal(recording.points.length, tries.length); + const written = JSON.stringify(recording.points); + for (const secret of [canary, admin.userId, admin.token, saveId, '203.0.113', 'Mozilla', 'PROPFIND', '?', 'octocat']) { + assert.ok(!written.includes(secret), secret); + } + assert.ok(recording.points.every(point => TEMPLATES.has(field(point, 'route')))); +}); + +test('sitePoint writes nothing outside the allowlist and refuses values outside its sets', () => { + const good = { release: 'unknown', route: '/admin/saves/:id/download', method: 'GET', status: '2xx', code: 'none', app: 'rise', ms: 12 }; + assert.deepEqual(sitePoint(good), { + indexes: ['site.request'], + blobs: ['site.request', 'unknown', '/admin/saves/:id/download', 'GET', '2xx', 'none', 'rise'], + doubles: [1, 12], + }); + assert.deepEqual(sitePoint({ ...good, release: '3d16ebe' }).blobs[1], '3d16ebe'); + const extra = sitePoint({ ...good, userId: 'user-canary', ip: '203.0.113.7', userAgent: 'Mozilla/5.0', query: 'q=1' }); + assert.ok(!JSON.stringify(extra).match(/user-canary|203\.0\.113|Mozilla|q=1/), 'fields the allowlist does not name are dropped'); + for (const bad of [ + { method: 'PATCH' }, { method: 'get' }, { status: '1xx' }, { status: '200' }, { app: 'evil' }, { app: 'Rise' }, + { route: '/admin/saves/abc?x=1' }, { route: '/Admin/People' }, { route: 'admin/' }, { route: '/admin/saves/a%2Fb/download' }, + { code: 'not found' }, { code: 'Error: D1_ERROR' }, { code: '' }, { release: 'main' }, { release: '3D16EBE' }, { release: '3d16ebee1cf2' }, + { ms: -1 }, { ms: Number.NaN }, { ms: '12' }, { route: undefined }, { app: null }, + ]) { + assert.equal(sitePoint({ ...good, ...bad }), null, JSON.stringify(bad)); + } +}); + +test('the allowlist file is the contract shape, and the code reads every field it lists', () => { + assert.equal(ALLOWLIST.version, 1); + assert.equal(ALLOWLIST.dataset, 'syberlabs_events'); + assert.deepEqual(Object.keys(ALLOWLIST.events), ['site.request']); + assert.equal(SPEC.owner, 'Seth'); + assert.equal(SPEC.decision, 'Are account saves working for each consumer app?'); + assert.deepEqual(SPEC.blobs.map(b => b.name), ['release', 'route', 'method', 'status', 'code', 'app']); + assert.deepEqual(SPEC.blobs.find(b => b.name === 'method').values, ['GET', 'HEAD', 'POST', 'OPTIONS', 'other']); + assert.deepEqual(SPEC.blobs.find(b => b.name === 'status').values, ['2xx', '3xx', '4xx', '5xx']); + assert.deepEqual(SPEC.blobs.find(b => b.name === 'app').values, ['omni', 'rise', 'sketch', 'site', 'none']); + assert.deepEqual(SPEC.doubles, ['count', 'ms']); + // At most 20 blobs and 20 doubles per data point (Analytics Engine limits); the event takes one blob. + assert.ok(1 + SPEC.blobs.length <= 20 && SPEC.doubles.length <= 20); +}); + +test('the EVENTS binding exists in the production configuration only', async () => { + const toml = await read('../../wrangler.toml'); + const tables = [...toml.matchAll(/^\[\[([^\]]+)\]\]\n((?:[^[\n].*\n?)*)/gm)].map(([, name, body]) => ({ name, body })); + const analytics = tables.filter(t => t.name.endsWith('analytics_engine_datasets')); + assert.deepEqual(analytics.map(t => t.name), ['analytics_engine_datasets'], 'top level only: never under env.preview'); + assert.match(analytics[0].body, /^binding = "EVENTS"$/m); + assert.match(analytics[0].body, new RegExp(`^dataset = "${ALLOWLIST.dataset}"$`, 'm')); + assert.doesNotMatch(toml, /\[\[?env\.[^\]]*analytics/); +}); diff --git a/telemetry/events.json b/telemetry/events.json new file mode 100644 index 0000000..f031df7 --- /dev/null +++ b/telemetry/events.json @@ -0,0 +1,19 @@ +{ + "version": 1, + "dataset": "syberlabs_events", + "events": { + "site.request": { + "owner": "Seth", + "decision": "Are account saves working for each consumer app?", + "blobs": [ + { "name": "release" }, + { "name": "route" }, + { "name": "method", "values": ["GET", "HEAD", "POST", "OPTIONS", "other"] }, + { "name": "status", "values": ["2xx", "3xx", "4xx", "5xx"] }, + { "name": "code" }, + { "name": "app", "values": ["omni", "rise", "sketch", "site", "none"] } + ], + "doubles": ["count", "ms"] + } + } +} diff --git a/wrangler.toml b/wrangler.toml index 7e18068..7be96e6 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -17,6 +17,12 @@ binding = "DB" database_name = "syberlabs-accounts" database_id = "039dab7a-ea89-44b9-a516-8509bf2848f3" +# site.request counts (MasterMind RFC 0004, telemetry.v1). Production only: Analytics Engine bindings do not inherit +# and preview declares none, so staging and preview deployments write nothing. The first write creates the dataset. +[[analytics_engine_datasets]] +binding = "EVENTS" +dataset = "syberlabs_events" + # vars and d1_databases do not inherit, so preview repeats both (RFC-0002 8.4.3). [env.preview.vars] ORIGIN = "https://staging.syberlabs.io"