diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d366fee5..4fe540ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ permissions: contents: read env: - WDL_CLI_PACKAGE: "@wdl-dev/cli@1.8.1" + WDL_CLI_PACKAGE: "@wdl-dev/cli@1.9.0" concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 41718e15..d64817ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- Added bounded Workflow definition pagination, with per-worker limits of 1024 retained names and 1 MiB of definition data. Large metadata snapshots fail before transfer instead of materializing an unbounded namespace view. Update CLI clients to `1.9.0` or later before rolling Control; CI and Quick Start now pin `1.9.0`. +- Bounded Workflow deletion preflight and batched token-fenced cleanup with internal continuation and delete-lock renewal. Large cleanup requests may return `workflow_lifecycle_check_incomplete` and be retried without losing completed cleanup work. +- Accounted retained, active/detached, and in-flight replay bytes under a 64 MiB Runtime budget; saturation is retryable and never becomes a fresh step claim. Persisted result limits apply consistently to list, get, and status; restart rejects invalid stored params before changing state. +- Bounded Workflow instance-list pages to 8 MiB of JSON while preserving payloads, order, and cursor continuation; Control enforces the same response cap and a five-second read deadline without re-encoding successful pages. Update Workflows before system-runtime; no state migration is required. +- Added `system_runtime_desired_count` to Terraform, preserving the default of one while allowing all service families to scale to zero for maintenance. + ## wdl.20260906.1 - 2026-09-07 - Updated workerd and Workers types to `1.20260906.1` and `5.20260906.1`, advancing the maximum compatibility date to `2026-09-13`; upstream runtime behavior, experimental flags, static dates, and autogates are unchanged. diff --git a/README.md b/README.md index 2a8a0238..13f71863 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ The platform is split into seven app services plus shared state: | `d1-runtime` | D1 SQLite execution with owner leases and supervised workerd process. | | `do-runtime` | Durable Object native facets, SQLite storage, alarms, owner leases, and WebSockets. | | `scheduler` | Cron, queue, and workflow tick dispatch. | -| `workflows` | Workflows V2 state machine, DB 2 owner, and DO alarm delivery backend. | +| `workflows` | Workflow state machine, DB 2 owner, and DO alarm delivery backend. | Valkey/Redis uses a deliberate logical split: @@ -172,7 +172,7 @@ compose bind-mounts `./dist` over the image's built configs. ```bash git clone https://github.com/wdl-dev/wdl.git cd wdl -npm install -g @wdl-dev/cli@1.8.1 +npm install -g @wdl-dev/cli@1.9.0 npm ci npm install --ignore-scripts --prefix examples/hello-jsonc npm run compile:workerd:local diff --git a/README.zh.md b/README.zh.md index 8af181a6..b7674521 100644 --- a/README.zh.md +++ b/README.zh.md @@ -75,7 +75,7 @@ WDL 首先是基础设施项目：operator 运行自己的 platform，tenant 用 | `d1-runtime` | D1 SQLite execution，使用 owner lease 和 supervisor 管理 workerd process。 | | `do-runtime` | Durable Object native facet、SQLite storage、alarm、owner lease 和 WebSocket。 | | `scheduler` | cron、queue 和 workflow tick dispatch。 | -| `workflows` | Workflows V2 state machine、DB 2 owner 和 DO alarm delivery backend。 | +| `workflows` | Workflow state machine、DB 2 owner 和 DO alarm delivery backend。 | Valkey/Redis 使用明确的逻辑切分： @@ -104,7 +104,7 @@ Clone 仓库，安装 standalone `wdl` CLI 和仓库依赖，编译本地 worker ```bash git clone https://github.com/wdl-dev/wdl.git cd wdl -npm install -g @wdl-dev/cli@1.8.1 +npm install -g @wdl-dev/cli@1.9.0 npm ci npm install --ignore-scripts --prefix examples/hello-jsonc npm run compile:workerd:local diff --git a/control/bundle.js b/control/bundle.js index 5a11b960..307c932f 100644 --- a/control/bundle.js +++ b/control/bundle.js @@ -20,6 +20,7 @@ import { } from "shared-workerd-compat-flags"; import { normalizeBindings, validateBindings } from "control-bindings"; import { parseWorkerdDependencyVersion } from "control-lib"; +import { WORKFLOW_DEFINITIONS_MAX_COUNT, workflowDeclarationsFit } from "control-workflow-definitions"; import { assertCanonicalBase64 } from "base64.js"; import PACKAGE_JSON_SOURCE from "wdl-package-json-source"; @@ -196,6 +197,9 @@ export function normalizeWorkflows(workflows) { if (!Array.isArray(workflows)) { throw new Error("workflows must be an array of entries"); } + if (workflows.length > WORKFLOW_DEFINITIONS_MAX_COUNT) { + throw new BundleConfigError(413, "workflow_definitions_too_large", "Workflow declaration count exceeds the per-worker limit"); + } const out = []; const seenNames = new Set(); const seenBindings = new Set(); @@ -207,7 +211,7 @@ export function normalizeWorkflows(workflows) { throw new BundleConfigError( 400, "workflow_script_name_unsupported", - "workflows script_name is not supported in WDL Workflows V2" + "workflows script_name is not supported in WDL Workflows" ); } if (typeof entry.name !== "string" || !WORKFLOW_NAME_RE.test(entry.name)) { @@ -256,6 +260,9 @@ export function normalizeWorkflows(workflows) { } out.push({ name: entry.name, binding: entry.binding, className }); } + if (!workflowDeclarationsFit(out)) { + throw new BundleConfigError(413, "workflow_definitions_too_large", "Workflow declarations exceed the per-worker byte limit"); + } return out; } diff --git a/control/handlers/delete.js b/control/handlers/delete.js index 0a359f8a..d5046c49 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -340,7 +340,7 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log, assertActiveVersionRetained(collected); let workflowBlocker = null; try { - await assertWorkflowDeleteAllowed({ ns, worker: name, allowCleanup: true, requestId }); + await assertWorkflowDeleteAllowed({ ns, worker: name, allowCleanup: true, requestId, lockToken }); } catch (err) { if (err instanceof ControlAbort && err.code === "workflow_instances_active") { workflowBlocker = err; diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 9a93eee7..18de6ed0 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -1,3 +1,4 @@ +import { readWorkflowDefinitionAdmission, workflowDeclarationsFit, workflowDefinitionUpdatesFit } from "control-workflow-definitions"; import { jsonResponse, jsonError, readJsonBody, formatError, requireControlLog, requireControlRedis, @@ -1252,10 +1253,15 @@ async function materializeCommittedMetadata(iso, { ns, name, prepared, resolvedD const workflowDefUpdates = []; if (Array.isArray(committedMeta.workflows) && committedMeta.workflows.length) { const defsKey = workflowDefsKey(ns, name); - const existingDefRaws = await iso.hMGet( + const admission = await readWorkflowDefinitionAdmission( + iso, defsKey, committedMeta.workflows.map((workflow) => workflow.name) ); + if (admission.status !== 1) { + throw new DeployAbort(500, "workflow_definition_corrupt", { message: "Stored Workflow definitions exceed their bounds" }); + } + const existingDefRaws = admission.values; for (let index = 0; index < committedMeta.workflows.length; index += 1) { const workflow = committedMeta.workflows[index]; const existingDef = parsePersistedWorkflowDef(workflow.name, existingDefRaws[index]); @@ -1272,6 +1278,9 @@ async function materializeCommittedMetadata(iso, { ns, name, prepared, resolvedD JSON.stringify({ workflowKey, className: workflow.className }), ]); } + if (!workflowDeclarationsFit(committedMeta.workflows) || !workflowDefinitionUpdatesFit(admission, workflowDefUpdates.map(([, field, value]) => [field, value]))) { + throw new DeployAbort(413, "workflow_definitions_too_large", { message: "Workflow definitions exceed the per-worker count or byte limit" }); + } } let doStorageId = null; diff --git a/control/handlers/versions.js b/control/handlers/versions.js index dc571bd1..4c2c5cf6 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -100,7 +100,7 @@ async function handleDelete({ ns, name, version, principal, requestId }) { let result; try { await assertWorkflowDeleteAllowed({ - ns, worker: name, version, allowCleanup: true, requestId, + ns, worker: name, version, allowCleanup: true, requestId, lockToken, }); if (!await renewDeleteLock(redis, ns, name, lockToken)) { throw new VersionDeleteError(409, "deleting", deleteLockExpiredDetails(ns, name, version)); diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index b8a0d133..fe0344b5 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -1,16 +1,29 @@ import { parseBundleMeta, workflowDefsKey } from "control-lib"; import { ControlAbort, + WORKFLOWS_INTERNAL_TIMEOUT_MS, codedErrorLogFields, controlAbortResponse, errorMessage, jsonError, jsonResponse, postWorkflowsInternal, + readWorkflowInstancesResponse, requireControlLog, requireControlRedis, } from "control-shared"; import { bundleKey, routesKey } from "shared-worker-contract"; +import { bytesToBase64, canonicalBase64ToBytes } from "base64.js"; +import { utf8ByteLength } from "shared-utf8"; +import { + WORKFLOW_DEFINITION_CURSOR_MAX_BYTES, + WORKFLOW_DEFINITION_PAGE_MAX_BYTES, + WORKFLOW_DEFINITION_PAGE_READ_MAX_BYTES, + WORKFLOW_DEFINITION_PAGE_MAX_WORKERS, + readWorkflowDefinitionSnapshot, + readWorkflowRoutePage, + workflowDeclarationsFit, +} from "control-workflow-definitions"; import { BINDING_NAME_RE, WORKER_NAME_RE, @@ -23,6 +36,8 @@ import { const LIFECYCLE_ACTIONS = new Set(["pause", "resume", "restart", "terminate"]); const MAX_WORKFLOW_SNAPSHOT_ATTEMPTS = 2; +const cursorEncoder = new TextEncoder(); +const cursorDecoder = new TextDecoder("utf-8", { fatal: true }); /** * @typedef {import("shared-redis").RedisClient} RedisClient @@ -47,6 +62,7 @@ const MAX_WORKFLOW_SNAPSHOT_ATTEMPTS = 2; * }} WorkflowRequest * @typedef {{ workflow: WorkflowEntry, request: WorkflowRequest }} ResolvedWorkflow * @typedef {{ error: string, message?: unknown, [key: string]: unknown }} UpstreamErrorBody + * @typedef {{ scan: string, worker?: string, name?: string, fingerprint?: string, definitions?: string }} DefinitionCursor */ /** @param {WorkflowsHandlerArgs} args */ @@ -79,7 +95,7 @@ async function handleInner({ method, url, ns, subPath, requestId }) { const log = requireControlLog(); const deps = { redis }; if (method === "GET" && subPath.length === 0) { - const body = await listWorkflowDefinitions(deps, ns); + const body = await listWorkflowDefinitions(deps, ns, listOptions(url)); log("info", "workflows_listed", { request_id: requestId, namespace: ns, @@ -93,7 +109,7 @@ async function handleInner({ method, url, ns, subPath, requestId }) { const workflow = await resolveWorkflow(deps, ns, worker, workflowName); if (method === "GET" && subPath.length === 3) { - const body = await callWorkflowsRust("instances", { + const { body, responseBytes } = await callWorkflowsRust("instances", { ...workflow.request, options: listOptions(url), requestId, @@ -105,13 +121,15 @@ async function handleInner({ method, url, ns, subPath, requestId }) { workflow: workflowName, count: Array.isArray(body.instances) ? body.instances.length : 0, }); - return jsonResponse(200, body); + return new Response(/** @type {BodyInit | null} */ (responseBytes), { + headers: { "content-type": "application/json" }, + }); } if (subPath.length >= 4) { const instanceId = decodePathSegment(subPath[3], "workflow instance id"); if (method === "GET" && subPath.length === 4) { - const body = await callWorkflowsRust("status", { + const { body } = await callWorkflowsRust("status", { ...workflow.request, instanceId, options: statusOptions(url), @@ -135,7 +153,7 @@ async function handleInner({ method, url, ns, subPath, requestId }) { message: `Workflow ${ns}/${worker}/${workflowName} is not exported by the active worker version`, }); } - const body = await callWorkflowsRust(action, { + const { body } = await callWorkflowsRust(action, { ...workflow.request, instanceId, requestId, @@ -160,100 +178,169 @@ async function handleInner({ method, url, ns, subPath, requestId }) { /** * @param {{ redis: RedisClient }} deps * @param {string} ns + * @param {Record} options */ -async function listWorkflowDefinitions({ redis }, ns) { - return await redis.session(async (session) => - listWorkflowDefinitionsFromSession({ redis: session }, ns)); +async function listWorkflowDefinitions({ redis }, ns, options) { + const limit = options.limit ?? 100; + if (typeof limit !== "number" || !Number.isInteger(limit) || limit < 1 || limit > 1000) { + throw new ControlAbort(400, "invalid_request", { message: "Workflow definition limit must be in [1, 1000]" }); + } + const cursor = decodeDefinitionCursor(options.cursor); + try { + return await redis.session(async (session) => + listWorkflowDefinitionsFromSession({ redis: session }, ns, limit, cursor)); + } catch (err) { + if (err instanceof ControlAbort) throw err; + requireControlLog()("error", "workflow_metadata_unavailable", { namespace: ns, error_message: errorMessage(err) }); + throw new ControlAbort(500, "workflow_metadata_unavailable", { namespace: ns, message: "Workflow metadata is unavailable" }); + } +} + +/** @param {Uint8Array} bytes */ +function base64Url(bytes) { + return bytesToBase64(bytes).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/, ""); +} + +/** @param {unknown} value */ +async function definitionFingerprint(value) { + return base64Url(new Uint8Array(await crypto.subtle.digest("SHA-256", cursorEncoder.encode(JSON.stringify(value))))); +} + +/** @param {DefinitionCursor} cursor */ +function encodeDefinitionCursor(cursor) { + return base64Url(cursorEncoder.encode(JSON.stringify(cursor))); +} + +/** @param {unknown} raw @returns {DefinitionCursor} */ +function decodeDefinitionCursor(raw) { + if (raw === undefined || raw === "") return { scan: "0" }; + try { + if (typeof raw !== "string" || raw.length > WORKFLOW_DEFINITION_CURSOR_MAX_BYTES || !/^[A-Za-z0-9_-]+$/.test(raw)) throw new Error("cursor"); + const base64 = raw.replaceAll("-", "+").replaceAll("_", "/"); + const parsed = JSON.parse(cursorDecoder.decode(canonicalBase64ToBytes(base64 + "=".repeat((4 - base64.length % 4) % 4)))); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || typeof parsed.scan !== "string" || !/^(0|[1-9][0-9]{0,19})$/.test(parsed.scan) || BigInt(parsed.scan) > 18446744073709551615n) throw new Error("cursor"); + if (parsed.worker !== undefined && (!isValidWorkerName(parsed.worker) || typeof parsed.fingerprint !== "string")) throw new Error("cursor"); + if (parsed.name !== undefined && (parsed.worker === undefined || !isValidWorkflowName(parsed.name) || typeof parsed.definitions !== "string")) throw new Error("cursor"); + return parsed; + } catch { + throw new ControlAbort(400, "invalid_request", { message: "Workflow definition cursor is invalid" }); + } } /** * @param {{ redis: RedisSession }} deps * @param {string} ns + * @param {number} limit + * @param {DefinitionCursor} cursor */ -async function listWorkflowDefinitionsFromSession({ redis }, ns) { +async function listWorkflowDefinitionsFromSession({ redis }, ns, limit, cursor) { for (let attempt = 0; attempt < MAX_WORKFLOW_SNAPSHOT_ATTEMPTS; attempt += 1) { - const routes = await redis.hGetAll(routesKey(ns)); - /** @type {Array<[string, string]>} */ - const routeEntries = []; - for (const [worker, activeVersion] of Object.entries(routes)) { - if (typeof activeVersion === "string" && activeVersion) routeEntries.push([worker, activeVersion]); - } - if (routeEntries.length === 0) return { namespace: ns, workflows: [] }; - const [metaRaws, defsRaws] = await readWorkflowListRaws({ redis }, ns, routeEntries); - const currentRoutes = await redis.hGetAll(routesKey(ns)); - /** @type {Array | undefined>} */ - const metas = new Array(routeEntries.length); - for (let index = 0; index < routeEntries.length; index += 1) { - const [worker, activeVersion] = routeEntries[index]; - if (currentRoutes[worker] === activeVersion) { - metas[index] = workflowBundleMeta(ns, worker, activeVersion, metaRaws[index]); + const routePage = await readWorkflowRoutePage(redis, routesKey(ns), cursor.scan); + const entries = Object.entries(routePage.routes).filter(([, version]) => version).toSorted(([a], [b]) => a.localeCompare(b)); + const fingerprint = await definitionFingerprint([routePage.next, entries]); + if (cursor.fingerprint && cursor.fingerprint !== fingerprint) break; + /** @type {ListedWorkflowEntry[]} */ + const workflows = []; + /** @type {DefinitionCursor} */ + let progress = { ...cursor, fingerprint }; + /** @type {DefinitionCursor | null} */ + let next = routePage.next === "0" ? null : { scan: routePage.next }; + let bytes = utf8ByteLength(JSON.stringify({ namespace: ns, workflows: [], cursor: "" })) + WORKFLOW_DEFINITION_CURSOR_MAX_BYTES; + let workers = 0; + let inputBytes = 0; + let changed = false; + for (const [worker, version] of entries) { + const order = cursor.worker === undefined ? 1 : worker.localeCompare(cursor.worker); + if (order < 0 || (order === 0 && cursor.name === undefined)) continue; + if (workers === WORKFLOW_DEFINITION_PAGE_MAX_WORKERS || workflows.length === limit) { next = progress; break; } + workers += 1; + const snapshot = await readWorkflowListSnapshot(redis, ns, worker, version, entries.length); + if (snapshot.status === 2) { changed = true; break; } + if (snapshot.status === -1) { + throw new ControlAbort(413, "workflow_listing_metadata_too_large", { namespace: ns, worker, message: "Worker metadata exceeds the Workflow listing read budget" }); + } + if (snapshot.status !== 1) { + throw new ControlAbort(500, "corrupt_meta", { namespace: ns, worker, message: "Workflow listing metadata exceeds its read bounds" }); + } + if (inputBytes + snapshot.bytes > WORKFLOW_DEFINITION_PAGE_READ_MAX_BYTES) { + if (inputBytes === 0) { + throw new ControlAbort(413, "workflow_listing_metadata_too_large", { namespace: ns, worker, message: "Worker metadata exceeds the Workflow listing read budget" }); + } + next = progress; + break; } + inputBytes += snapshot.bytes; + const meta = workflowBundleMeta(ns, worker, version, snapshot.metaRaw); + const definitions = buildWorkerWorkflowDefinitions(ns, worker, version, meta, snapshot.defs); + const defsFingerprint = await definitionFingerprint(definitions); + if (order === 0 && cursor.name !== undefined && cursor.definitions !== defsFingerprint) { changed = true; break; } + let pageFull = false; + for (const definition of definitions) { + if (order === 0 && cursor.name !== undefined && definition.name.localeCompare(cursor.name) <= 0) continue; + const size = utf8ByteLength(JSON.stringify(definition)) + (workflows.length > 0 ? 1 : 0); + if (workflows.length === limit || bytes + size > WORKFLOW_DEFINITION_PAGE_MAX_BYTES) { + if (workflows.length === 0) throw new ControlAbort(500, "corrupt_meta", { namespace: ns, worker, message: "Workflow definition exceeds the page byte limit" }); + next = progress; + pageFull = true; + break; + } + workflows.push(definition); + bytes += size; + progress = { scan: cursor.scan, fingerprint, worker, name: definition.name, definitions: defsFingerprint }; + } + if (pageFull) break; + progress = { scan: cursor.scan, fingerprint, worker }; } - if (!sameRouteSnapshot(routes, currentRoutes)) continue; - return buildWorkflowDefinitionList( - ns, - routeEntries, - /** @type {Record[]} */ (metas), - defsRaws - ); + if (changed) continue; + const after = await readWorkflowRoutePage(redis, routesKey(ns), cursor.scan); + const afterEntries = Object.entries(after.routes).filter(([, version]) => version).toSorted(([a], [b]) => a.localeCompare(b)); + if (await definitionFingerprint([after.next, afterEntries]) !== fingerprint) continue; + return { namespace: ns, workflows, cursor: next === null ? null : encodeDefinitionCursor(next) }; } throw new ControlAbort(503, "workflow_metadata_contention", { - message: "Workflow metadata changed while it was being read", + message: "Workflow metadata changed; restart definition listing without a cursor", namespace: ns, }); } /** * @param {string} ns - * @param {Array<[string, string]>} routeEntries - * @param {Array>} metas - * @param {Array>} defsRaws + * @param {string} worker + * @param {string} activeVersion + * @param {Record} meta + * @param {Record} defsRaw */ -function buildWorkflowDefinitionList(ns, routeEntries, metas, defsRaws) { +function buildWorkerWorkflowDefinitions(ns, worker, activeVersion, meta, defsRaw) { /** @type {ListedWorkflowEntry[]} */ const workflows = []; - for (let i = 0; i < routeEntries.length; i += 1) { - const [worker, activeVersion] = routeEntries[i]; - const meta = metas[i]; - const activeByName = new Map(); - for (const workflow of workflowsFromMeta(meta)) { - activeByName.set(workflow.name, workflow); - workflows.push({ - namespace: ns, - worker, - activeVersion, - name: workflow.name, - binding: workflow.binding, - className: workflow.className, - workflowKey: workflow.workflowKey, - }); - } - const defs = parseWorkflowDefs(defsRaws[i], { ns, worker }); - for (const [name, def] of Object.entries(defs)) { - if (activeByName.has(name)) continue; - workflows.push({ - namespace: ns, - worker, - activeVersion, - name, - binding: null, - className: def.className, - workflowKey: def.workflowKey, - retired: true, - }); - } + const activeNames = new Set(); + for (const workflow of workflowsFromMeta(meta)) { + activeNames.add(workflow.name); + workflows.push({ + namespace: ns, + worker, + activeVersion, + name: workflow.name, + binding: workflow.binding, + className: workflow.className, + workflowKey: workflow.workflowKey, + }); } - const sortedWorkflows = workflows.toSorted((a, b) => - a.worker.localeCompare(b.worker) || - a.name.localeCompare(b.name) - ); - return { namespace: ns, workflows: sortedWorkflows }; -} - -/** @param {Record} left @param {Record} right */ -function sameRouteSnapshot(left, right) { - const keys = Object.keys(left); - return keys.length === Object.keys(right).length && keys.every((key) => right[key] === left[key]); + const defs = parseWorkflowDefs(defsRaw, { ns, worker }); + for (const [name, def] of Object.entries(defs)) { + if (activeNames.has(name)) continue; + workflows.push({ + namespace: ns, + worker, + activeVersion, + name, + binding: null, + className: def.className, + workflowKey: def.workflowKey, + retired: true, + }); + } + return workflows.toSorted((a, b) => a.name.localeCompare(b.name)); } /** @@ -422,7 +509,7 @@ function workflowBundleMeta(ns, worker, version, raw) { }); const workflows = meta.workflows; if (workflows !== undefined) { - if (!Array.isArray(workflows)) { + if (!workflowDeclarationsFit(workflows) || !Array.isArray(workflows)) { throw corruptWorkflowEntries(ns, worker, version); } const names = new Set(); @@ -456,32 +543,18 @@ function corruptWorkflowEntries(ns, worker, version) { }); } -/** - * @param {{ redis: RedisSession }} deps - * @param {string} ns - * @param {Array<[string, string]>} routeEntries - * @returns {Promise<[Array, Array>]>} - */ -async function readWorkflowListRaws({ redis }, ns, routeEntries) { +/** @param {RedisSession} redis @param {string} ns @param {string} worker @param {string} version @param {number} workerCount */ +async function readWorkflowListSnapshot(redis, ns, worker, version, workerCount) { try { - const snapshot = await redis.hGetManyAndHGetAllMany( - routeEntries.map(([worker, activeVersion]) => [ - bundleKey(ns, worker, activeVersion), - "__meta__", - ]), - routeEntries.map(([worker]) => workflowDefsKey(ns, worker)) - ); - return [snapshot.fields, snapshot.hashes]; + return await readWorkflowDefinitionSnapshot(redis, workflowDefsKey(ns, worker), bundleKey(ns, worker, version), { + key: routesKey(ns), worker, version, + }); } catch (err) { requireControlLog()("error", "workflow_metadata_unavailable", { - namespace: ns, - worker_count: routeEntries.length, - error_message: errorMessage(err), + namespace: ns, worker_count: workerCount, error_message: errorMessage(err), }); throw new ControlAbort(500, "workflow_metadata_unavailable", { - message: "Workflow metadata is unavailable", - namespace: ns, - worker_count: routeEntries.length, + message: "Workflow metadata is unavailable", namespace: ns, worker_count: workerCount, }); } } @@ -547,9 +620,11 @@ function isActiveWorkflowMeta(entry) { /** * @param {string} endpoint * @param {WorkflowRequest} body - * @returns {Promise>} + * @returns {Promise<{ body: Record, responseBytes: Uint8Array | null }>} */ async function callWorkflowsRust(endpoint, body) { + /** @type {Uint8Array | null} */ + let responseBytes = null; const { response, body: parsed } = await postWorkflowsInternal({ endpoint: `workflows/${endpoint}`, body, @@ -558,7 +633,12 @@ async function callWorkflowsRust(endpoint, body) { logFields: { endpoint, }, - timeoutMs: null, + timeoutMs: endpoint === "instances" ? WORKFLOWS_INTERNAL_TIMEOUT_MS : null, + readBody: endpoint === "instances" ? async (response, signal) => { + const result = await readWorkflowInstancesResponse(response, signal); + responseBytes = result.bytes; + return result.body; + } : undefined, }); if (!response.ok) { if (isUpstreamErrorBody(parsed)) { @@ -587,7 +667,7 @@ async function callWorkflowsRust(endpoint, body) { message: "Workflow backend returned an invalid response", }); } - return /** @type {Record} */ (parsed); + return { body: /** @type {Record} */ (parsed), responseBytes }; } /** diff --git a/control/index.js b/control/index.js index 90ee9d86..6915c7a9 100644 --- a/control/index.js +++ b/control/index.js @@ -94,7 +94,7 @@ const NS_SCOPED_KINDS = new Set([ ]); const PLATFORM_VERSION = platformVersionFromSource(WDL_VERSION_SOURCE); -const MIN_CLI_VERSION = "1.8.0"; +const MIN_CLI_VERSION = "1.9.0"; /** * @param {Request} request diff --git a/control/shared.js b/control/shared.js index 4fe6a3b1..440a4708 100644 --- a/control/shared.js +++ b/control/shared.js @@ -38,7 +38,11 @@ import { } from "shared-observability"; import { WORKFLOWS_INTERNAL_TIMEOUT_MS, + WORKFLOW_LIFECYCLE_MAX_PAGES, + WORKFLOW_LIFECYCLE_TIMEOUT_MS, createPostWorkflowsInternal, + readWorkflowInstancesResponse, + readWorkflowLifecycleResponse, } from "control-workflows-client"; import { ControlAbort, @@ -106,6 +110,7 @@ export { }; export { runOptimistic, withOptimisticRetries }; export { readJsonBody }; +export { WORKFLOWS_INTERNAL_TIMEOUT_MS, readWorkflowInstancesResponse }; /** @param {unknown} value @returns {value is Record} */ function isRecord(value) { @@ -609,48 +614,81 @@ export async function recordCleanupIntentOrWarn({ } /** - * @param {{ ns: string, worker: string, version?: string, allowCleanup?: boolean, requestId?: string | null }} args + * @param {{ ns: string, worker: string, version?: string, allowCleanup?: boolean, requestId?: string | null, lockToken?: string }} args */ -export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefined, allowCleanup = false, requestId = null }) { +export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefined, allowCleanup = false, requestId = null, lockToken }) { const context = { namespace: ns, worker, ...(version ? { version } : {}), }; - const { response, body } = await postWorkflowsInternal({ - endpoint: "workflows/lifecycle/check-delete", - body: { - ns, - worker, - ...(version ? { version } : {}), - ...(allowCleanup ? { allowCleanup: true } : {}), - }, - requestId, - logEvent: "workflow_lifecycle_check_failed", - logFields: context, - errorDetails: context, - unavailableMessage: "Workflow lifecycle check is unavailable", - requestFailedMessage: "Workflow lifecycle check failed", - // The lifecycle scan is unbounded by namespace size. Preserve its - // pre-consolidation behavior instead of imposing the ordinary short - // control-to-workflows request timeout. - timeoutMs: null, - }); - if (!response.ok) { - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow lifecycle check failed", - ...context, - upstream_status: response.status, - upstream_error: isRecord(body) && typeof body.error === "string" ? body.error : null, - }); - } - if (!isRecord(body) || typeof body.allowed !== "boolean") { - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow lifecycle check returned an invalid response", - ...context, + const deadline = Date.now() + WORKFLOW_LIFECYCLE_TIMEOUT_MS; + let cursor; + for (let page = 0; page < WORKFLOW_LIFECYCLE_MAX_PAGES; page += 1) { + if (Date.now() >= deadline) break; + if (lockToken) { + /** @type {ReturnType | undefined} */ + let renewalTimeout; + try { + const renewed = await Promise.race([ + renewDeleteLock(requireControlRedis(), ns, worker, lockToken), + new Promise((resolve) => { + renewalTimeout = setTimeout(() => resolve(null), Math.max(0, deadline - Date.now())); + }), + ]); + if (renewed === null || Date.now() >= deadline) break; + if (!renewed) throw new ControlAbort(409, "deleting", deleteLockExpiredDetails(ns, worker, version)); + } finally { + if (renewalTimeout !== undefined) clearTimeout(renewalTimeout); + } + } + if (Date.now() >= deadline) break; + const { response, body } = await postWorkflowsInternal({ + endpoint: "workflows/lifecycle/check-delete", + body: { + ns, + worker, + ...(version ? { version } : {}), + ...(allowCleanup ? { allowCleanup: true } : {}), + ...(cursor === undefined ? {} : { cursor }), + }, + requestId, + logEvent: "workflow_lifecycle_check_failed", + logFields: context, + errorDetails: context, + unavailableMessage: "Workflow lifecycle check is unavailable", + requestFailedMessage: "Workflow lifecycle check failed", + timeoutMs: WORKFLOWS_INTERNAL_TIMEOUT_MS, + deadlineMs: deadline, + deadlineErrorCode: "workflow_lifecycle_check_incomplete", + readBody: readWorkflowLifecycleResponse, }); - } - if (body.allowed !== true) { + if (!response.ok) { + throw new ControlAbort(503, "workflow_internal_dispatch_failed", { + message: "Workflow lifecycle check failed", + ...context, + upstream_status: response.status, + upstream_error: isRecord(body) && typeof body.error === "string" ? body.error : null, + }); + } + if (!isRecord(body) || typeof body.allowed !== "boolean" || (body.allowed && body.cursor != null)) { + throw new ControlAbort(503, "workflow_internal_dispatch_failed", { + message: "Workflow lifecycle check returned an invalid response", + ...context, + }); + } + if (body.allowed) return; + if (Array.isArray(body.blockers) && body.blockers.length === 0) { + if (body.cursor === undefined) break; + if (typeof body.cursor !== "string" || !/^[1-9][0-9]*$/.test(body.cursor) || body.cursor === cursor) { + throw new ControlAbort(503, "workflow_internal_dispatch_failed", { + message: "Workflow lifecycle check returned an invalid cursor", + ...context, + }); + } + cursor = body.cursor; + continue; + } throw new ControlAbort(409, "workflow_instances_active", { message: version ? `${ns}/${worker}/${version} has active workflow instances` @@ -660,6 +698,10 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi blockers: Array.isArray(body.blockers) ? body.blockers : [], }); } + throw new ControlAbort(503, "workflow_lifecycle_check_incomplete", { + message: "Workflow lifecycle check is incomplete; retry the deletion request", + ...context, + }); } /** diff --git a/control/workflow-definitions.js b/control/workflow-definitions.js new file mode 100644 index 00000000..b414562a --- /dev/null +++ b/control/workflow-definitions.js @@ -0,0 +1,131 @@ +import { decodeBulk } from "shared-redis"; +import { utf8ByteLength } from "shared-utf8"; + +export const WORKFLOW_DEFINITIONS_MAX_COUNT = 1024; +export const WORKFLOW_DEFINITIONS_MAX_BYTES = 1024 * 1024; +export const WORKFLOW_LIST_META_MAX_BYTES = 8 * 1024 * 1024; +export const WORKFLOW_DEFINITION_PAGE_MAX_BYTES = 8 * 1024 * 1024; +export const WORKFLOW_DEFINITION_PAGE_READ_MAX_BYTES = 16 * 1024 * 1024; +export const WORKFLOW_DEFINITION_PAGE_MAX_WORKERS = 16; +export const WORKFLOW_DEFINITION_CURSOR_MAX_BYTES = 2048; + +export const WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT = ` +if redis.call('HGET', KEYS[3], ARGV[4]) ~= ARGV[5] then return {2, false, {}, 0} end +if redis.call('HLEN', KEYS[1]) > tonumber(ARGV[1]) then return {0, false, {}, 0} end +local fields = redis.call('HKEYS', KEYS[1]) +local bytes = 0 +for _, field in ipairs(fields) do + bytes = bytes + #field + redis.call('HSTRLEN', KEYS[1], field) + if bytes > tonumber(ARGV[2]) then return {0, false, {}, 0} end +end +local meta_bytes = redis.call('HSTRLEN', KEYS[2], '__meta__') +if meta_bytes > tonumber(ARGV[3]) then return {-1, false, {}, 0} end +local meta = redis.call('HGET', KEYS[2], '__meta__') +return {1, meta, redis.call('HGETALL', KEYS[1]), bytes + meta_bytes} +`; + +export const WORKFLOW_DEFINITION_ADMISSION_SCRIPT = ` +local count = redis.call('HLEN', KEYS[1]) +if count > tonumber(ARGV[1]) then return {0, count, 0, {}} end +local bytes = 0 +for _, field in ipairs(redis.call('HKEYS', KEYS[1])) do + bytes = bytes + #field + redis.call('HSTRLEN', KEYS[1], field) + if bytes > tonumber(ARGV[2]) then return {0, count, bytes, {}} end +end +local values = {} +for i = 3, #ARGV do values[#values + 1] = redis.call('HGET', KEYS[1], ARGV[i]) end +return {1, count, bytes, values} +`; + +export const WORKFLOW_ROUTE_PAGE_SCRIPT = ` +local page = redis.call('HSCAN', KEYS[1], ARGV[1], 'COUNT', 32) +if #page[2] > 1024 then return {0, false, {}} end +local bytes = 0 +for _, value in ipairs(page[2]) do + bytes = bytes + #value + if bytes > 131072 then return {0, false, {}} end +end +return {1, page[1], page[2]} +`; + +/** @param {unknown} raw @returns {Record} */ +function decodeHash(raw) { + if (!Array.isArray(raw) || raw.length % 2 !== 0) throw new Error("Invalid Workflow hash snapshot"); + const result = Object.create(null); + for (let index = 0; index < raw.length; index += 2) { + const key = decodeBulk(raw[index]); + const value = decodeBulk(raw[index + 1]); + if (key == null || value == null) throw new Error("Invalid Workflow hash entry"); + result[key] = value; + } + return result; +} + +/** + * @param {import("shared-redis").RedisClient | import("shared-redis").RedisSession} redis + * @param {string} defsKey + * @param {string} metaKey + * @param {{key: string, worker: string, version: string}} route + */ +export async function readWorkflowDefinitionSnapshot(redis, defsKey, metaKey, route) { + const raw = await redis.eval(WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT, + [defsKey, metaKey, route.key], + [String(WORKFLOW_DEFINITIONS_MAX_COUNT), String(WORKFLOW_DEFINITIONS_MAX_BYTES), String(WORKFLOW_LIST_META_MAX_BYTES), route.worker, route.version]); + if (!Array.isArray(raw) || raw.length !== 4) throw new Error("Invalid Workflow definition snapshot"); + const metaRaw = decodeBulk(raw[1]); + const defs = decodeHash(raw[2]); + const bytes = Number(raw[3]); + return { status: Number(raw[0]), metaRaw, defs, bytes }; +} + +/** + * @param {import("shared-redis").RedisSession} redis + * @param {string} key + * @param {string[]} names + */ +export async function readWorkflowDefinitionAdmission(redis, key, names) { + const raw = await redis.eval(WORKFLOW_DEFINITION_ADMISSION_SCRIPT, [key], + [String(WORKFLOW_DEFINITIONS_MAX_COUNT), String(WORKFLOW_DEFINITIONS_MAX_BYTES), ...names]); + if (!Array.isArray(raw) || raw.length !== 4 || !Array.isArray(raw[3])) throw new Error("Invalid Workflow admission snapshot"); + const values = raw[3].map((value) => { + const decoded = decodeBulk(value); + if (decoded === undefined) throw new Error("Missing Workflow admission reply"); + return decoded; + }); + if (raw[0] === 1 && values.length !== names.length) throw new Error("Workflow admission reply count mismatch"); + return { status: Number(raw[0]), count: Number(raw[1]), bytes: Number(raw[2]), values }; +} + +/** + * @param {import("shared-redis").RedisClient | import("shared-redis").RedisSession} redis + * @param {string} key + * @param {string} cursor + */ +export async function readWorkflowRoutePage(redis, key, cursor) { + const raw = await redis.eval(WORKFLOW_ROUTE_PAGE_SCRIPT, [key], [cursor]); + if (!Array.isArray(raw) || raw.length !== 3 || raw[0] !== 1) { + throw new Error("Workflow route scan page exceeds its bounds"); + } + const next = decodeBulk(raw[1]); + if (next == null) throw new Error("Invalid Workflow route cursor"); + return { next, routes: decodeHash(raw[2]) }; +} + +/** @param {{count: number, bytes: number, values: (string | null)[]}} snapshot @param {Array<[string, string]>} updates */ +export function workflowDefinitionUpdatesFit(snapshot, updates) { + let { count, bytes } = snapshot; + for (let index = 0; index < updates.length; index += 1) { + const [name, value] = updates[index]; + const old = snapshot.values[index]; + if (old == null) { count += 1; bytes += utf8ByteLength(name); } + else bytes -= utf8ByteLength(old); + bytes += utf8ByteLength(value); + } + return count <= WORKFLOW_DEFINITIONS_MAX_COUNT && bytes <= WORKFLOW_DEFINITIONS_MAX_BYTES; +} + +/** @param {unknown} declarations */ +export function workflowDeclarationsFit(declarations) { + return Array.isArray(declarations) && declarations.length <= WORKFLOW_DEFINITIONS_MAX_COUNT && + utf8ByteLength(JSON.stringify(declarations)) <= WORKFLOW_DEFINITIONS_MAX_BYTES; +} diff --git a/control/workflows-client.js b/control/workflows-client.js index 6f41bdd1..dfb48479 100644 --- a/control/workflows-client.js +++ b/control/workflows-client.js @@ -1,12 +1,41 @@ import { errorMessage } from "shared-errors"; +import { readBoundedBytes } from "shared-bounded-body"; +import { discardResponseBody } from "shared-respond"; import { ControlAbort } from "control-errors"; export const WORKFLOWS_INTERNAL_TIMEOUT_MS = 5_000; +export const MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES = 8 * 1024 * 1024; +export const WORKFLOW_LIFECYCLE_MAX_PAGES = 16; +export const WORKFLOW_LIFECYCLE_TIMEOUT_MS = 10_000; +export const WORKFLOW_LIFECYCLE_RESPONSE_MAX_BYTES = 64 * 1024; + +const workflowResponseDecoder = new TextDecoder("utf-8", { fatal: true }); + +/** @param {Response} response @param {number} maxBytes @param {AbortSignal} [signal] */ +async function readWorkflowResponse(response, maxBytes, signal) { + try { + const bytes = await readBoundedBytes(response, maxBytes, signal); + return { body: JSON.parse(workflowResponseDecoder.decode(bytes)), bytes }; + } catch (err) { + await discardResponseBody(response); + throw err; + } +} + +/** @param {Response} response @param {AbortSignal} [signal] */ +export async function readWorkflowInstancesResponse(response, signal) { + return await readWorkflowResponse(response, MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES, signal); +} + +/** @param {Response} response @param {AbortSignal} [signal] */ +export async function readWorkflowLifecycleResponse(response, signal) { + return (await readWorkflowResponse(response, WORKFLOW_LIFECYCLE_RESPONSE_MAX_BYTES, signal)).body; +} /** * @typedef {{ fetch: typeof fetch }} WorkflowBackend * @typedef {(level: string, event: string, fields?: Record) => void} WorkflowClientLogger - * @typedef {"unavailable" | "request_failed"} WorkflowTransportFailure + * @typedef {"unavailable" | "request_failed" | "deadline"} WorkflowTransportFailure */ /** @@ -20,6 +49,7 @@ export const WORKFLOWS_INTERNAL_TIMEOUT_MS = 5_000; * logEvent: string, * logFields?: Record, * timeoutMs: number | null, + * deadlineMs?: number, * makeError: (failure: WorkflowTransportFailure) => Error, * readBody?: (response: Response, signal?: AbortSignal) => Promise, * }} args @@ -35,6 +65,7 @@ export async function postWorkflowsInternalRequest({ logEvent, logFields = {}, timeoutMs, + deadlineMs, makeError, readBody = async (response) => await response.json().catch(() => null), }) { @@ -42,29 +73,53 @@ export async function postWorkflowsInternalRequest({ throw makeError("unavailable"); } + /** @type {AbortSignal | undefined} */ + let signal; + let deadlineOwnsTimeout = false; + let timeoutDetected = false; try { + if (timeoutMs === undefined) throw new TypeError("Workflow timeoutMs must be explicitly set"); const requestHeaders = new Headers(headers()); if (typeof requestId === "string" && requestId) { requestHeaders.set("x-request-id", requestId); } - const signal = timeoutMs === null ? undefined : AbortSignal.timeout(timeoutMs); + const now = Date.now(); + let requestTimeoutMs = timeoutMs; + // Delayed rejection must retain the deadline selected for this request. + if (deadlineMs !== undefined && (timeoutMs === null || deadlineMs - now <= timeoutMs)) { + deadlineOwnsTimeout = true; + requestTimeoutMs = Math.max(0, deadlineMs - now); + } + const deadline = requestTimeoutMs === null ? null : now + requestTimeoutMs; + if (deadlineMs !== undefined && now >= deadlineMs) { + timeoutDetected = true; + throw new DOMException("Workflow request deadline expired", "TimeoutError"); + } + signal = requestTimeoutMs === null ? undefined : AbortSignal.timeout(requestTimeoutMs); const response = await workflows.fetch(`http://workflows/internal/${endpoint}`, { method: "POST", headers: requestHeaders, body: JSON.stringify(body), ...(signal === undefined ? {} : { signal }), }); + const responseBody = await readBody(response, signal); + signal?.throwIfAborted(); + if (deadline !== null && Date.now() >= deadline) { + timeoutDetected = true; + throw new DOMException("Workflow backend request timed out", "TimeoutError"); + } return { response, - body: await readBody(response, signal), + body: responseBody, }; } catch (err) { - log?.("error", logEvent, { + const deadlineExpired = deadlineOwnsTimeout && (signal?.aborted || timeoutDetected); + log?.(deadlineExpired ? "warn" : "error", logEvent, { ...logFields, ...(typeof requestId === "string" && requestId ? { request_id: requestId } : {}), error_message: errorMessage(err), }); - throw makeError("request_failed"); + throw makeError(deadlineExpired ? "deadline" : "request_failed"); } } @@ -85,6 +140,8 @@ export function createPostWorkflowsInternal({ getWorkflows, headers, getLog = () * logFields?: Record, * errorDetails?: Record, * timeoutMs: number | null, + * deadlineMs?: number, + * deadlineErrorCode?: string, * unavailableMessage?: string, * requestFailedMessage?: string, * readBody?: (response: Response, signal?: AbortSignal) => Promise, @@ -98,6 +155,8 @@ export function createPostWorkflowsInternal({ getWorkflows, headers, getLog = () logFields = {}, errorDetails = {}, timeoutMs, + deadlineMs, + deadlineErrorCode = "workflow_internal_dispatch_failed", unavailableMessage = "Workflow backend is unavailable", requestFailedMessage = "Workflow backend request failed", readBody, @@ -112,12 +171,13 @@ export function createPostWorkflowsInternal({ getWorkflows, headers, getLog = () logEvent, logFields, timeoutMs, + deadlineMs, readBody, - makeError: (failure) => new ControlAbort(503, "workflow_internal_dispatch_failed", { + makeError: (failure) => new ControlAbort(503, failure === "deadline" ? deadlineErrorCode : "workflow_internal_dispatch_failed", { ...errorDetails, message: failure === "unavailable" ? unavailableMessage - : requestFailedMessage, + : failure === "deadline" ? "Workflow request deadline expired" : requestFailedMessage, }), }); }; diff --git a/docs/compatibility.md b/docs/compatibility.md index 45cb15fa..1ef5702c 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -112,7 +112,7 @@ using compatibility flags in platform workers. | Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql` including R*Tree virtual tables, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-managed public WebSocket forwarding, storage ids, cleanup tombstones, and bundle-owned `preserve`/`restart` session policy. `preserve` is the default; `restart` closes superseded public WebSockets with `1012` and lazily aborts stale facets without deleting SQLite. Deployment-level `DO_PREVENT_EVICTION` defaults to `true`, keeping host actors resident; explicit `false` permits idle actor reconstruction while preserving SQLite and quiescent hibernatable socket state. | WDL DO identity, owner routing, session policy, cleanup, and residency policy are WDL-managed rather than Cloudflare migration-compatible. `restart` matches Cloudflare's default, where deploying new code restarts every Durable Object and disconnects its WebSockets; `preserve` is the WDL default and has no Cloudflare equivalent. A later `preserve` promotion can supersede restart work not yet observed. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. Actor-local JavaScript state does not survive eviction. Tenant `ctx.storage.deleteAll()` is a WDL best-effort KV/SQL/Workflows sequence rather than Cloudflare's atomic storage operation; rejection may leave a partial result, and concurrent mutation is unsupported. Alarm mutation spans object SQLite and Workflows DB 2; after backend mutation starts, rejected `setAlarm()` or `deleteAlarm()` has unknown final state and requires a successful operation-specific retry. `getAlarm()` repair is best effort and does not confirm backend state. Native `ctx.abort(..., { retryAlarm: false })` does not suppress retries for WDL's Workflows-backed alarm dispatch. Current workerd can still lose an in-flight hibernatable WebSocket application send or close across actor eviction, so eviction remains explicit opt-in; workerd 2026-08-25 fixes the former in-flight auto-response loss. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | | Queues producer/consumer | Partial | Worker queue handler API surface in loaded workers. | Runtime producer facade writes through redis-proxy DB 1; scheduler owns consumer dispatch, retry, DLQ, orphan, delayed queues, and batch splitting for oversized dispatch bodies. | `max_batch_timeout_ms` is configuration metadata, not a true aggregation window; dispatch concurrency is scheduler-owned. | `max_concurrency` is rejected. | | Cron triggers | Supported | `scheduled()` handler surface in workers. | Control stores cron config; scheduler owns indexed discovery, cron-slot buckets, due dispatch, and repairable projections. | Control and scheduler use JS/Rust `croner` engines and keep their behavior documented through tests and module docs. | New scheduler dispatch paths still require their own Redis lease/fence audit. | -| Workflows | Partial | User workflow class code runs inside loaded workers when dispatched by runtime. | workflows owns DB 2 instance state, step/event/sleep commits, runtime-observed `step.do` DAG edges including `Promise.all` siblings, generation/run-token fencing, lifecycle APIs, progress callbacks, and scheduler tick. | WDL Workflows V2 has WDL-specific payload semantics and terminal failure rules; permanent `step.do` failure is terminal for the run even if caught, and one step may record at most 1000 dependency edges. | WDL Workflows V2 is not Cloudflare Workflows parity. Native workerd `WorkflowInstance.delete()` and `Workflow.deleteBatch()` are not exposed by WDL's custom facade; `locationHint`, `script_name`, cross-worker workflows, and Cloudflare's source-AST visualizer are also unsupported. | +| Workflows | Partial | User workflow class code runs inside loaded workers when dispatched by runtime. | workflows owns DB 2 instance state, step/event/sleep commits, runtime-observed `step.do` DAG edges including `Promise.all` siblings, generation/run-token fencing, lifecycle APIs, progress callbacks, and scheduler tick. | Payloads are JSON data under WDL byte caps; permanent `step.do` failure is terminal for the run even if caught, and one step may record at most 1000 dependency edges. | Native workerd `WorkflowInstance.delete()` and `Workflow.deleteBatch()` are not exposed by WDL's custom facade; `locationHint`, `script_name`, cross-worker workflows, and Cloudflare's source-AST visualizer are also unsupported. | | Service bindings | Supported | workerd service binding JSRPC and fetch dispatch. | WDL resolves target worker metadata, cold-loads immutable versions, propagates request ids where available, and enforces namespace/action ACL through control metadata. | Frozen-version service targets do not evict active siblings by design. | Object-style custom RPC has no general generated env facade; use class-based `WorkerEntrypoint` methods with `this.env` when binding facades are required. | | Platform bindings | Supported | workerd named entrypoint/JSRPC mechanics. | WDL expands ACL-checked platform bindings from control metadata. | Platform bindings are WDL-specific. | Platform bindings are not a Cloudflare tenant portability feature. | | Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget for user vars/secrets plus runtime-injected binding env values and binding-scoped Workflow identity props during deploy/secret mutation, accounts for V8 two-byte string storage for non-Latin-1 strings, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index 4d973290..554e7146 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -59,7 +59,7 @@ Bundled workerd 允许 `Fetcher` 和 Durable Object class stub 在不启用 expe | Durable Objects | Partial | Native Durable Object class execution、facet identity、支持 R*Tree virtual table 的 SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-managed public WebSocket forwarding、storage id、cleanup tombstone，以及由 bundle 拥有的 `preserve`/`restart` session policy。`preserve` 是默认值；`restart` 会以 `1012` 关闭已 supersede 的 public WebSocket，并在不删除 SQLite 的前提下 lazy abort stale facet。部署级 `DO_PREVENT_EVICTION` 默认是 `true`，会保持 host actor resident；显式 `false` 允许 idle actor 重建，同时保留 SQLite 与静默 hibernatable socket state。 | WDL DO identity、owner routing、session policy、cleanup 和 residency policy 由 WDL 管理，不兼容 Cloudflare migration 模型。`restart` 与 Cloudflare 的默认行为一致——部署新代码会重启每个 Durable Object 并断开其 WebSocket；`preserve` 是 WDL 默认值，Cloudflare 没有对应选项。后续 `preserve` promotion 可以 supersede 尚未被观察到的 restart work。 | 只支持同 worker class；不支持 `script_name`、rename/delete migrations 和平台级 WebSocket session recovery。Actor-local JavaScript state 不会跨 eviction 保留。Tenant `ctx.storage.deleteAll()` 是 WDL best-effort KV/SQL/Workflows sequence，不是 Cloudflare atomic storage operation；reject 可能留下部分结果，且不支持 concurrent mutation。Alarm mutation 横跨 object SQLite 与 Workflows DB 2；backend mutation 开始后，reject 的 `setAlarm()` 或 `deleteAlarm()` 最终状态未知，需要成功完成与操作意图一致的重试。`getAlarm()` repair 只是 best effort，不确认 backend state。Native `ctx.abort(..., { retryAlarm: false })` 不会禁止 WDL Workflows-backed alarm retry。当前 workerd 仍可能在 actor eviction 期间静默丢失在途 hibernatable WebSocket application send 或 close，因此 eviction 保持显式 opt-in；workerd 2026-08-25 已修复此前的在途 auto-response 遗留。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | | Queues producer/consumer | Partial | loaded worker 中的 queue handler surface。 | Runtime producer facade 经 redis-proxy 写 DB 1；scheduler 负责 consumer dispatch、retry、DLQ、orphan、delayed queues，并会拆分过大的 dispatch body batch。 | `max_batch_timeout_ms` 是配置 metadata，不是真正 aggregation window；dispatch concurrency 由 scheduler 拥有。 | `max_concurrency` 被拒绝。 | | Cron triggers | Supported | worker 的 `scheduled()` handler surface。 | Control 存 cron config；scheduler 拥有 indexed discovery、cron-slot bucket、due dispatch 和可修复 projection。 | Control 和 scheduler 使用 JS/Rust `croner` 引擎，并通过测试和模块文档约束行为。 | 新增 scheduler dispatch 路径仍必须单独审计 Redis lease/fence。 | -| Workflows | Partial | 用户 workflow class code 在 runtime dispatch 时运行在 loaded worker 内。 | workflows 拥有 DB 2 instance state、step/event/sleep commit、runtime-observed `step.do` DAG edges（包括 `Promise.all` sibling）、generation/run-token fence、lifecycle API、progress callback 和 scheduler tick。 | WDL Workflows V2 的 payload 语义和 terminal failure 规则由 WDL 定义；永久失败的 `step.do` 即使被捕获也会让 run terminal failed；单个 step 最多记录 1000 条 dependency edge。 | WDL Workflows V2 不是 Cloudflare Workflows parity。WDL 自有 facade 不暴露原生 workerd 的 `WorkflowInstance.delete()` 和 `Workflow.deleteBatch()`；也不支持 `locationHint`、`script_name`、跨 worker workflow 和 Cloudflare 的源码 AST visualizer。 | +| Workflows | Partial | 用户 workflow class code 在 runtime dispatch 时运行在 loaded worker 内。 | workflows 拥有 DB 2 instance state、step/event/sleep commit、runtime-observed `step.do` DAG edges（包括 `Promise.all` sibling）、generation/run-token fence、lifecycle API、progress callback 和 scheduler tick。 | Payload 是受 WDL byte cap 约束的 JSON data；永久失败的 `step.do` 即使被捕获也会让 run terminal failed；单个 step 最多记录 1000 条 dependency edge。 | WDL 自有 facade 不暴露原生 workerd 的 `WorkflowInstance.delete()` 和 `Workflow.deleteBatch()`；也不支持 `locationHint`、`script_name`、跨 worker workflow 和 Cloudflare 的源码 AST visualizer。 | | Service bindings | Supported | workerd service binding JSRPC 和 fetch dispatch。 | WDL 解析 target worker metadata，cold-load immutable version，在可用时传播 request id，并通过 control metadata 做 namespace/action ACL。 | Frozen-version service target 按设计不 evict active siblings。 | 对象式自定义 RPC 不提供通用的 generated env facade；需要 binding facade 时，使用 class-based `WorkerEntrypoint` 方法中的 `this.env`。 | | Platform bindings | Supported | workerd named entrypoint/JSRPC 机制。 | WDL 从 control metadata 展开 ACL-checked platform bindings。 | Platform bindings 是 WDL-specific。 | 不是 Cloudflare tenant portability feature。 | | Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata，把 secret value 加密成 at-rest `WDL-ENC:` envelope；redis-proxy 在 cold-load 时解密；deploy/secret mutation 过程中按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 校验用户 vars/secrets、runtime 注入的 binding env value 和 binding-scoped Workflow identity props，并计入非 Latin-1 字符串的 V8 two-byte storage；worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理；at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | diff --git a/docs/modules/cli.md b/docs/modules/cli.md index b3433954..b64cd3a8 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -94,7 +94,12 @@ The maintained compatibility declaration for current `main` and its next release | WDL source line | Minimum supported CLI | CI-qualified CLI | | --- | --- | --- | -| Current `main` / next release | `1.8.0` | `1.8.1` | +| Current `main` / next release | `1.9.0` | `1.9.0` | + +Workflow definition pagination requires CLI `1.9.0` or later to pass opaque cursors +and display continuation after short or empty pages. Upgrade installed CLI clients +before deploying the paginated Control endpoint; upgrading WDL services does not +upgrade the CLI. The minimum is a support declaration for the full CLI surface. The CI-qualified version is the exact published package used by WDL's CLI integration job. These values are not @@ -108,8 +113,8 @@ minimum complete CLI is the earliest published CLI that can express all stable CLI-managed capabilities introduced by that WDL range; it is not the possibly lagging package that happened to run in release CI. Use the latest patch release at or above the listed boundary. A newer CLI can still expose commands that an older WDL release does -not implement, so server-side validation remains canonical. The final row remains open -until a later WDL release raises the minimum complete CLI. +not implement, so server-side validation remains canonical. This matrix covers +published WDL releases; the source-line declaration above covers unreleased changes. | WDL release range | Minimum complete CLI | Capability boundary | | --- | --- | --- | @@ -119,7 +124,7 @@ until a later WDL release raises the minimum complete CLI. | `wdl.20260724.1` - `wdl.20260801.1` | `1.6.0` | `workers_dev = false` and route URL reporting. | | `wdl.20260801.2` - `wdl.20260804.1` | None (`1.6.0` baseline) | Existing CLI commands remain usable, but the transient `durableObjectRollout` opt-in was Control-API-only and never had a published CLI spelling. | | `wdl.20260804.2` - `wdl.20260815.1` | `1.7.0` | `[wdl] session_policy` deploy support. | -| `wdl.20260817.1` and later | `1.8.0` | AI binding manifests and namespace AI provider management. | +| `wdl.20260817.1` - `wdl.20260906.1` | `1.8.0` | AI binding manifests and namespace AI provider management. | The CLI must treat the remaining discovery fields as diagnostics and defaults for user-facing guidance, not as a replacement for explicit user configuration. Missing @@ -193,7 +198,7 @@ Supported config surfaces: | `workers_dev` | `false` opts the worker out of the `.//` subdomain route while keeping pattern routes active; requires at least one `route`/`routes` entry. Defaults to enabled. | | `[triggers] crons` and `[[triggers.schedules]]` | UTC Cloudflare-compatible crons plus WDL timezone extension. | | `[[queues.producers]]` and `[[queues.consumers]]` | Producer and consumer metadata. `max_concurrency` is rejected. | -| `[[workflows]]` | Same-worker Workflows V2 bindings. | +| `[[workflows]]` | Same-worker Workflow bindings. | | `[ai]` | Declares one tenant binding name, for example `binding = "AI"`. Provider metadata and credentials remain namespace resources managed separately through `wdl ai`; they are not embedded in the bundle or inherited into selected environments. | `[[analytics_engine_datasets]]` is rejected at deploy at both top level and selected-env @@ -267,6 +272,9 @@ integration files marked `// @wdl-cli-integration`: - `tests/integration/route-demo.test.js` - `tests/integration/s3-cleanup.test.js` +CLI smoke tests cover definition and instance pagination against the platform, +including text/JSON output, short or empty continuation pages, and final cursors. + The CLI repository owns `[ai]` parsing, extension stripping, provider command tests, and the `examples/ai-agent-demo` packaging path. The platform-side runtime and Control contract is covered by `tests/integration/ai-binding.test.js`. diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index d8515fbf..5fb47998 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -58,11 +58,13 @@ CLI 可以展示： | WDL source line | 最低支持 CLI | CI-qualified CLI | | --- | --- | --- | -| 当前 `main` / 下一次 release | `1.8.0` | `1.8.1` | +| 当前 `main` / 下一次 release | `1.9.0` | `1.9.0` | + +Workflow definition 分页要求 CLI `1.9.0` 或更高版本，以传递 opaque cursor 并在短页或空页后显示 continuation。部署分页 Control endpoint 前应先升级已安装的 CLI client；升级 WDL service 不会升级 CLI。 最低版本是对完整 CLI surface 的支持声明；CI-qualified 版本是 WDL CLI integration job 使用的精确已发布 package。两者都不是 request negotiation 或执行门禁：普通 CLI 命令不会预检 `/whoami`，Control 也不会按 CLI 版本拒绝请求。`whoami` 和 `doctor` 可以比较当前 CLI 与 `minCliVersion` 并报告兼容结果；每个请求仍由服务端执行 canonical validation。 -已发布版本使用以下按能力推导的兼容矩阵。最低完整 CLI 是能够表达对应 WDL 区间所引入全部稳定 CLI 管理能力的最早已发布版本，不取决于 release CI 当时可能滞后的 package。应使用不低于表中边界的最新 patch release。较新的 CLI 仍可能提供旧 WDL 没有实现的命令，因此服务端 validation 继续作为 canonical contract。最后一行保持开放，直到后续 WDL release 提高最低完整 CLI 时再闭合。 +已发布版本使用以下按能力推导的兼容矩阵。最低完整 CLI 是能够表达对应 WDL 区间所引入全部稳定 CLI 管理能力的最早已发布版本，不取决于 release CI 当时可能滞后的 package。应使用不低于表中边界的最新 patch release。较新的 CLI 仍可能提供旧 WDL 没有实现的命令，因此服务端 validation 继续作为 canonical contract。此矩阵覆盖已发布的 WDL release；未发布变更使用上方 source-line 声明。 | WDL release range | 最低完整 CLI | 能力边界 | | --- | --- | --- | @@ -72,7 +74,7 @@ CLI 可以展示： | `wdl.20260724.1` - `wdl.20260801.1` | `1.6.0` | `workers_dev = false` 与 route URL reporting。 | | `wdl.20260801.2` - `wdl.20260804.1` | 无（基础命令为 `1.6.0`） | 既有 CLI 命令仍可用，但短暂存在的 `durableObjectRollout` opt-in 仅有 Control API，从未发布对应 CLI spelling。 | | `wdl.20260804.2` - `wdl.20260815.1` | `1.7.0` | `[wdl] session_policy` deploy 支持。 | -| `wdl.20260817.1` 及后续版本 | `1.8.0` | AI binding manifest 与 namespace AI provider management。 | +| `wdl.20260817.1` - `wdl.20260906.1` | `1.8.0` | AI binding manifest 与 namespace AI provider management。 | CLI 应把其余 discovery 字段当作 diagnostics 和 user-facing guidance 的默认值，而不是替代用户显式配置。可选 URL hint 缺失时，应展示为 unavailable，不应自行猜测。 @@ -124,7 +126,7 @@ WDL 遵循 Wrangler selected-env 继承规则： | `workers_dev` | `false` 让该 worker 退出 `.//` subdomain route，同时保留 pattern route；要求至少有一条 `route`/`routes`。默认启用。 | | `[triggers] crons` 和 `[[triggers.schedules]]` | Cloudflare-compatible UTC cron 加 WDL timezone extension。 | | `[[queues.producers]]` 和 `[[queues.consumers]]` | Producer 和 consumer metadata。`max_concurrency` 被拒绝。 | -| `[[workflows]]` | Same-worker Workflows V2 binding。 | +| `[[workflows]]` | Same-worker Workflow binding。 | | `[ai]` | 声明一个 tenant binding name，例如 `binding = "AI"`。Provider metadata 和 credential 是由 `wdl ai` 单独管理的 namespace resource，不会进入 bundle，也不会继承到 selected environment。 | `[[analytics_engine_datasets]]` 在 top level 和 selected-env level 都会被 deploy 拒绝。Unsupported field 不应在暗示 WDL 未实现的平台行为时被静默忽略。 @@ -174,4 +176,6 @@ Tail 是 live debug 路径，不是 audit storage。Tail protocol 细节见 [Log - `tests/integration/route-demo.test.js` - `tests/integration/s3-cleanup.test.js` +CLI smoke tests 直接验证平台的 definition 与 instance 分页，包括 text/JSON 输出、短页或空页的续接，以及最终 cursor。 + CLI 仓库负责 `[ai]` parsing、extension stripping、provider command tests 和 `examples/ai-agent-demo` packaging path；平台侧 runtime/Control 合同由 `tests/integration/ai-binding.test.js` 保护。 diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index d9e6d7d1..408426d8 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -289,7 +289,7 @@ Auth-specific contract: new delegated issue until the storage contract violation is repaired. Token list remains an operator repair surface and reports malformed `issue_templates` fields as invalid entries instead of failing the entire list. - Delegated namespace collision checks are best-effort in V1: unbound/full-plane + Delegated namespace collision checks are best-effort: unbound/full-plane tokens can still perform namespace-scoped writes that leave no auth-visible `ns` token record, while `namespaces` only reflects active workers. Routine delegated namespace workflows should use namespace-bound credentials for namespace-scoped @@ -368,9 +368,19 @@ Auth-specific contract: scheduler dispatch open for minutes. - All Control-to-Workflows internal POSTs use the canonical transport in `control/workflows-client.js`. Callers retain endpoint-specific timeout, non-2xx, and - response-body interpretation. Workflow management calls and the lifecycle delete scan - have no client-side timeout because the scan is unbounded by namespace size. DO-alarm - cleanup uses a five-second timeout. Workflow lifecycle blockers fail closed on service + response-body interpretation. Callers must explicitly select `timeoutMs`, even + with an absolute deadline; only `null` disables the per-call cap. + Instance lists have an 8 MiB response cap and a + five-second backend deadline covering fetch, body read, and JSON parsing. Successful + list JSON bytes are forwarded without re-serialization. Instance status/lifecycle + calls retain their timeout policy. Deletion preflight follows bounded internal pages + for at most 16 pages / ten seconds, with five-second per-page timeouts, 64 KiB response + caps, and token-scoped lock renewal. An unfinished page cannot authorize deletion; + total-budget expiry during a page returns + `workflow_lifecycle_check_incomplete`, while earlier per-call timeouts and + backend failures retain the dispatch-failure code. DO-alarm cleanup uses a + five-second timeout. + Workflow lifecycle blockers fail closed on service errors. - AUTH JSRPC errors or Redis explosions are control-plane failures and map to 503 fail-closed behavior, not tenant-visible authorization fallbacks. diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 151308b6..844db4cf 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -140,7 +140,7 @@ Auth 子合同： - Reserved namespace check 同时存在于 route/auth red line 和 role scope check；两者不能互相替代。 - `issue` 可以签发 `ns`、`platform`、`platform-observer` 和 `ops-observer` token。`token-issuer` 也可由 ops direct issue，但必须提供 camelCase `issueTemplates`，且其中每个 id 都要指向已有 delegated issue template。Auth 把这个 allowlist 存成 Redis `issue_templates` JSON array string，并拒绝 public `issue_templates` 输入，避免 API shape 和 storage shape 混用。`ops` 只能来自 bootstrap。Token plaintext 只生成一次、展示一次，并且只以 SHA-256 hash 形式存储。 - 内置 delegated issue template 包括用于 workshop pool 的 `wdl-chat-ns-pool`，以及用于 hosted CLI live integration 短期 namespace 的 `wdl-cli-integration`。 -- `token-issuer` 是 unbound role；除 `/whoami` self-introspection 外，它唯一 non-diagnostic action 是 `auth.delegated_token.issue`。Delegated issue 请求只接受 `{ template }`；caller-provided `kind`、`ns`、`label`、`expiresAt` 或 template allowlist 字段都会被拒。Auth 会在最终 issue-lock WATCH 内重新读取 issuer token record，确认它仍 active 且 allowlist 包含该 template，再应用 Auth 代码内定义的 template registry，并写入带 `created_by`、`issue_template`、`issue_template_version` metadata 的目标 token。Active quota 基于 `created_by + issue_template + expires_at` 计算，是 live credential capacity guard，不是 environment quota。生成的 namespace 是写入 token record 的普通 tenant namespace 字符串。Auth 会先用 `auth:delegated-issue-lock::` 串行化同一 issuer/template 的发放和 quota 计数，然后拒绝已经出现在 control 维护的 `namespaces` set 或任意已扫描 token record stored `ns` 中的 generated namespace candidate，不区分 token kind、issuer、template、是否过期或是否撤销。Quota 计算 fail-closed：同一 issuer/template 的 delegated token record 损坏会阻止新的 delegated issue，直到 storage contract violation 被修复。`GET /auth/tokens` 是 operator repair surface；单条 malformed `issue_templates` 会作为 invalid entry 列出，不会拖垮整个 list。V1 的 delegated namespace collision check 是 best-effort：unbound/full-plane token 仍能执行不会留下 auth-visible `ns` token record 的 namespace-scoped 写入，而 `namespaces` 只表示 active worker。常规 delegated namespace workflow 应使用 namespace-bound credential 做 namespace-scoped 写；未来的持久 namespace fact index 才是这个 residual risk 的完整修复。 +- `token-issuer` 是 unbound role；除 `/whoami` self-introspection 外，它唯一 non-diagnostic action 是 `auth.delegated_token.issue`。Delegated issue 请求只接受 `{ template }`；caller-provided `kind`、`ns`、`label`、`expiresAt` 或 template allowlist 字段都会被拒。Auth 会在最终 issue-lock WATCH 内重新读取 issuer token record，确认它仍 active 且 allowlist 包含该 template，再应用 Auth 代码内定义的 template registry，并写入带 `created_by`、`issue_template`、`issue_template_version` metadata 的目标 token。Active quota 基于 `created_by + issue_template + expires_at` 计算，是 live credential capacity guard，不是 environment quota。生成的 namespace 是写入 token record 的普通 tenant namespace 字符串。Auth 会先用 `auth:delegated-issue-lock::` 串行化同一 issuer/template 的发放和 quota 计数，然后拒绝已经出现在 control 维护的 `namespaces` set 或任意已扫描 token record stored `ns` 中的 generated namespace candidate，不区分 token kind、issuer、template、是否过期或是否撤销。Quota 计算 fail-closed：同一 issuer/template 的 delegated token record 损坏会阻止新的 delegated issue，直到 storage contract violation 被修复。`GET /auth/tokens` 是 operator repair surface；单条 malformed `issue_templates` 会作为 invalid entry 列出，不会拖垮整个 list。Delegated namespace collision check 是 best-effort：unbound/full-plane token 仍能执行不会留下 auth-visible `ns` token record 的 namespace-scoped 写入，而 `namespaces` 只表示 active worker。常规 delegated namespace workflow 应使用 namespace-bound credential 做 namespace-scoped 写；未来的持久 namespace fact index 才是这个 residual risk 的完整修复。 - `expiresAt` 必须是严格 ISO-8601 UTC、毫秒精度，并通过真实日历 round trip 校验。过期 token 在过期后第一次 verify 时 lazy collect：删除 active hash index，并写入 `expired_at`；主动 revoke 使用 `revoked_at`。 - 保留 token id `bootstrap` 是 infra-managed ops token。`BOOTSTRAP_TOKEN` 在 auth cold start 时 upsert，`revoke("bootstrap")` 被禁止，轮换方式是更新环境变量并 redeploy。Verify 会缓存已 ensure 的 bootstrap hash，但 Redis 丢失后可以重新 ensure，因此 bootstrap token 能恢复被 flush 的 auth store。 @@ -161,8 +161,9 @@ Auth 子合同： - Control 同样会在 watched revision-CAS loop 前加密 AI credential，并且 provider read/list API 永不返回 plaintext。 - Worker delete 先 commit Redis lifecycle state；异步 S3 cleanup enqueue 是 best-effort，失败时返回 warning。 - `s3-cleanup` system worker 会把 cleanup task 持久化在 D1；row 存在后由 cron replay 负责重试。S3 失败使用分钟级 exponential backoff，最高 30 分钟；大前缀 cleanup 每完成一个 S3 List/Delete page 就 checkpoint continuation token，因此正常分页进度不会消耗 failure attempts，也不会在 scheduler timeout 后从头开始。每次 run 只处理一页，所以超大 prefix 会跨多个 cron 或 queue dispatch 排空，而不是让单次 scheduler dispatch 持续数分钟。 -- 所有 Control-to-Workflows internal POST 都使用 `control/workflows-client.js` 的 canonical transport。Caller 继续持有 endpoint-specific timeout、non-2xx 和 response-body interpretation。Workflow 管理调用和 lifecycle delete scan 不设 client-side timeout，因为 scan 的规模不受 namespace 上限约束；DO-alarm cleanup 使用五秒 timeout。Workflow lifecycle blocker 在服务错误时 fail closed。 +- 所有 Control-to-Workflows internal POST 都使用 `control/workflows-client.js` 的 canonical transport。Caller 继续持有 endpoint-specific timeout、non-2xx 和 response-body interpretation。Instance list 使用 8 MiB response cap，五秒 backend deadline 覆盖 fetch、body read 和 JSON parse；成功的 list JSON bytes 不经重新序列化直接转发。Instance status/lifecycle 调用保留原有 timeout 策略。Deletion preflight 最多续接 16 页 / 十秒，每页最多五秒、response 最多 64 KiB，并续租相同 token 的 delete lock；未完成 page 不能授权删除，在途 page 耗尽总预算时返回 `workflow_lifecycle_check_incomplete`，更早的单次请求 timeout 和 backend failure 保留 dispatch-failure code。DO-alarm cleanup 使用五秒 timeout。Workflow lifecycle blocker 在服务错误时 fail closed。 - AUTH JSRPC 错误或 Redis 爆炸属于控制面失败，映射为 503 fail closed，而不是 tenant-visible authorization fallback。 +- 即使传入 absolute deadline，Control-to-Workflows caller 也必须显式选择 `timeoutMs`；只有 `null` 表示禁用单次请求 cap。 ## 安全边界 diff --git a/docs/modules/infra.md b/docs/modules/infra.md index de6f36c1..5052777f 100644 --- a/docs/modules/infra.md +++ b/docs/modules/infra.md @@ -197,6 +197,14 @@ Stateful storage: changes should document which services can use `FARGATE_SPOT`; stateful runtimes and singleton control loops should stay on on-demand Fargate unless their interruption semantics are re-reviewed. +- All seven Terraform service families expose desired-count inputs, including + `system_runtime_desired_count` with default `1`. Full-stop maintenance may set all + counts to `0`; operators must still drain old tasks and converge selected revisions + before restoring capacity in the release's required order. + Setting only system-runtime to zero also stops Control/Auth and `__system__` + Workers. Admin APIs, deployment, and system-targeted dispatch are unavailable. + Other pools may still serve existing tenant workloads; this is not a + full-platform stop. - In addition to the Fargate task memory limit, D1 and DO workerd containers set explicit container memory hard limits. DO also reserves memory for its local redis-proxy sidecar. diff --git a/docs/modules/infra.zh.md b/docs/modules/infra.zh.md index 4b78c90f..85bea79e 100644 --- a/docs/modules/infra.zh.md +++ b/docs/modules/infra.zh.md @@ -105,6 +105,8 @@ Stateful storage： - D1/DO 使用 owner lease、monotonic generation fence 和本地 drain/renew。超过 1 个 task 时需要稳定的 per-replica storage identity，并确保 supervisor drain/renew 只走私有本地入口，不能通过 service alias 打到其他 replica。 - 普通 D1/DO task 丢失会退回到 lease expiry，再由其他 replica takeover；graceful rollout 应优先走 supervisor drain，在 child workerd process 退出前释放 ownership。 - Terraform 在 ECS Fargate 上运行这套应用 stack 的服务。修改 capacity policy 时应记录哪些服务可以使用 `FARGATE_SPOT`；stateful runtime 和 singleton control loop 除非重新评估 interruption 语义，否则应保持 on-demand Fargate。 +- 七个 Terraform service family 都提供 desired-count 输入，包括默认值为 `1` 的 `system_runtime_desired_count`。全停机维护可将所有 count 设为 `0`；operator 仍须清退旧 task、确认目标 revision 已收敛，再按 release 要求的顺序恢复容量。 +- 只将 system-runtime 设为零也会停止 Control/Auth 与 `__system__` Worker execution，因此 admin API、deploy 和发往系统 Worker 的 dispatch 不可用。其他 runtime pool 仍可能继续服务已有 tenant workload；这不等于全平台停机。 - 除了 Fargate task memory limit，D1 和 DO 的 workerd container 还会设置显式 container memory hard limit。DO 还会给本地 redis-proxy sidecar 保留内存。 - do-runtime 的 `DO_PREVENT_EVICTION` 默认是 `true`，会让 host actor resident，避免当前 workerd 的 actor eviction 中断在途 hibernatable WebSocket 操作。显式 `false` 会为已经验证的 workload 启用 eviction；它不能替代 container memory hard limit。 - 所有 Terraform Fargate service 都使用 start-before-stop rolling replacement：`maximum_percent = 200`、`minimum_healthy_percent = 100`。Wire-compatible D1/DO release 可以安全重叠 router task，由 owner lease、generation fence 和 supervisor drain 控制状态交接；wire transition 无法安全重叠时，以 release Changelog 的额外 gate 覆盖该默认策略。scheduler 的重叠由各 dispatch path 的 Redis claim/fence 保护。 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index d13aa848..95b8a44f 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -202,6 +202,10 @@ Common rules: reset on scrape. The shared registry adds the `wdl_` prefix and `_total` counter suffix in Prometheus output. `completed` is a lease-lifecycle outcome, not a binding success result; binding-operation metrics own success/error classification. +- Workflow replay separates retained cache, active/detached state, and in-flight read + reservations. Its working-set/high-water gauges and `saturated` outcome are defined + in [Workflow observability](workflows.md#observability); do not add overlapping active + and retained views together or treat accounted bytes as RSS. - Rust `request_complete` logs report integer `duration_ms` values so log fields stay stable across services; Prometheus duration summaries keep their floating point values. diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index 2c4ef39c..1b817c40 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -101,6 +101,7 @@ WDL 在 JS workerd tier 和 Rust 服务中使用同一套可观测性策略： - JS 和 Rust observability 实现刻意共享 metric prefix `wdl`、request metric families `requests` / `request_duration_ms` / `request_errors`、cardinality warning threshold `100`、Prometheus content type `text/plain; version=0.0.4; charset=utf-8`，以及结构化日志的 key 顺序、level priority、stream routing 和 timestamp shape；共享 fixture `tests/fixtures/observability-contract.json` pin 住这些值，但不引入 runtime observability owner。 - redis-proxy 用 `kv_value_bytes` summary 记录 KV payload size，label 只有有界的 `service`/`operation`/`kind`。它记录 value、metadata 和 raw batch byte count，用来判断是否需要 large-value offload；namespace、key 和 object identity 不进入 metric label。 - Runtime KV admission 发布带有界 `service`/`outcome` label 的 `kv_read_capacity_events`，固定 outcome 为 `acquired`、`saturated`、`completed`、`deadline`、`setup_error`。`kv_read_in_flight_bytes` 表示当前预留 wire bytes；`kv_read_in_flight_high_water_bytes` 是进程生命周期最大值，不会随 scrape 重置。共享 registry 会在 Prometheus 输出中添加 `wdl_` prefix 和 counter 的 `_total` suffix。`completed` 是 lease lifecycle outcome，不代表 binding 成功；binding-operation metrics 拥有 success/error 分类。 +- Workflow replay 区分 retained cache、active/detached state 与在途 read reservation。Working-set/high-water gauges 和 `saturated` outcome 见 [Workflow 可观测性](workflows.zh.md#可观测性)；active 与 retained 视图重叠，不能重复相加，也不能把计费字节当作 RSS。 - Rust `request_complete` log 输出整数 `duration_ms`，让各服务日志字段保持稳定；Prometheus duration summary 仍保留浮点值。 - JS `MetricsRegistry` 在单个 metric name 达到 100 个 series 时输出一次结构化 `metric_cardinality_warning` 日志，之后会丢弃该 metric 的全新 series，但继续更新已有 series。Rust `MetricStore` 当前仍保持同一 warning-only tripwire。该 warning 携带 metric name、观测到的 series 数和配置 limit；tenant-specific 细节本来就不应进入 label。因为这条 warning 由 metrics registry 输出，所以不会被 `LOG_LEVEL` 抑制。 - `*_max` 是单独的 gauge family，不是 Prometheus summary family 下的额外 sample。Summary 只能输出 `_count`、`_sum` 和 quantile sample。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 721ffc13..4135f278 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -396,6 +396,13 @@ upstream flags during cold load. ## Observability +Workflow replay publishes retained, active, detached, and in-flight read byte views, +plus total working-set and process-lifetime high-water gauges. The 64 MiB working-set +budget includes retained/detached serialized records and replay response reservations; +it is not an RSS measurement. Saturation uses the existing retryable dispatch failure +path. See [Workflow observability](workflows.md#observability) for exact metric names +and overlap/accounting rules. + Runtime emits request logs and metrics for loading, binding operations, AI pool state, `redis-proxy` calls, workflow replay cache, loader evictions, and dispatch envelopes. Tail worker emits structured stdout for console/exception capture and forwards to `wdl tail` only diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 736f5869..e55abe6b 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -116,6 +116,8 @@ Runtime 可以把 Redis bundle metadata 视为 control-authored，但 materializ ## 可观测性 +Workflow replay 输出 retained、active、detached、在途 read bytes，以及总 working-set 和进程级 high-water gauges。64 MiB working-set 预算覆盖 retained/detached serialized record 与 replay response reservation，不是 RSS 测量；容量不足沿现有 retryable dispatch failure 路径处理。指标名与视图重叠规则见 [Workflow 可观测性](workflows.zh.md#可观测性)。 + Runtime 为 loading、binding operation、AI pool state、`redis-proxy` call、workflow replay cache、loader eviction 和 dispatch envelope 输出日志/metrics。Tail worker 总是为 console/exception capture 输出结构化 stdout；只有匹配的 active tail session 存在时才转发到 `wdl tail`。该 buffered tail pipeline 不导出 tracing span；WDL 没有 streaming-tail span sink。`tail_worker_user_spans` 已废弃，启用它也不会提供该 sink。 KV response admission 暴露 `wdl_kv_read_capacity_events_total{service,outcome}`，固定 outcome 为 `acquired`、`saturated`、`completed`、`deadline` 和 `setup_error`，并暴露当前值 `wdl_kv_read_in_flight_bytes{service}` 及进程生命周期 high-water `wdl_kv_read_in_flight_high_water_bytes{service}` gauge。Scrape 不会重置 high-water；只有 Runtime task 重启才会清零。`completed` 表示 lease 在 deadline 前释放，不表示 value parse 或 binding operation 成功；后者由 binding-operation metrics 拥有。 diff --git a/docs/modules/workflows.md b/docs/modules/workflows.md index 948be3eb..28fb981e 100644 --- a/docs/modules/workflows.md +++ b/docs/modules/workflows.md @@ -14,8 +14,8 @@ under `runtime/dispatch/workflow-*.js`. Control parses workflow metadata and own deploy-time workflow definition keys. This module doc is the current workflows design reference. -The V2 name distinguishes the current DAG-capable engine from the earlier test-only V1 -engine. New environments should be treated as greenfield schema-3 workflows state. +New deployments initialize schema 3 in dedicated DB 2. Existing schema-2 state requires +the offline [migration procedure](#deployment--rollout-notes). workerd provides the user-code execution environment, `WorkflowEntrypoint` class shape, module loading, and the ability for runtime to invoke a workflow class in a frozen @@ -40,8 +40,30 @@ User-facing: Control / CLI: - `GET /ns//workflows` lists workflow definitions and uses `workflow.list`. + It accepts `limit` (default 100, maximum 1000) and an opaque `cursor`. A page + processes at most 16 workers, at most 16 MiB of source snapshots plus one bounded + lookahead, and returns at most 8 MiB of JSON. Results are + sorted within the page, not globally across the namespace scan. Follow the + cursor until it is `null`, including after empty or short pages. If a route + segment or partially returned definition set changes, restart without a cursor + after `workflow_metadata_contention`; pagination is not a durable snapshot. + Concurrent namespace changes can repeat scan entries across pages; consumers + collecting a complete view should deduplicate by worker/name or restart. + Cursor input is limited to 2048 ASCII bytes. Route discovery treats + `HSCAN COUNT 32` as a hint and rejects pages above 512 field/value pairs or + 128 KiB of combined field/value bytes before transfer. Compact hash + encodings may return the whole hash regardless of COUNT. Increasing thresholds + such as `hash-max-listpack-entries` can therefore make listing fail closed; + WDL does not enforce a Valkey configuration value for those thresholds. - `GET /ns//workflows///instances` lists instances and uses - `workflow.read`. + `workflow.read`. `limit` defaults to 100 and accepts 1-1000; each page is also bounded + to 8 MiB of serialized JSON, including output/error payloads and the cursor envelope. + A byte-limited page can contain fewer than `limit` instances: continue with the returned + `cursor` until it is `null`, not until a short page. The rank cursor counts consumed + index entries, including missing/invisible instances, and resumes at the first + unreturned instance when the byte budget fills. It does not provide snapshot isolation + from concurrent index changes. Payload-ref errors for an instance being read fail + closed rather than silently omitting that instance. - `GET /ns//workflows///instances/` returns instance status and uses `workflow.read`. Optional query parameters use camelCase only: `includeSteps=true|false` includes step records, and `stepLimit=` limits returned @@ -88,6 +110,21 @@ identity. The hash retains retired names until whole-worker delete. Definition l enumerates that retired history for currently active workers; deploy and single-workflow status/lifecycle paths read only the names they need. +Each worker may retain at most 1024 definition names and 1 MiB of hash field/value +bytes. The active declaration array also has a 1 MiB JSON cap, including materialized +workflow keys. Deploy checks aggregate quota under the existing WATCH before committing; +only declared definition values are returned to the writer, while bounded size accounting +includes retired names. Quota rejection uses `workflow_definitions_too_large`. +Definition-list snapshots check cardinality and byte lengths inside Valkey before +returning values. The same snapshot atomically checks the worker's expected active +route and returns the byte total for page accounting; Control does not issue a separate +per-worker route read or re-encode the returned strings to count them. The final route +page fingerprint is still rechecked before returning the page. +Their per-version bundle metadata read budget is 8 MiB; exceeding it +returns `workflow_listing_metadata_too_large`, not an unbounded read. This is a listing +budget, not a new size limit on unrelated Worker code. Oversized retained definition +state fails closed rather than being truncated or silently adopted. + At normal startup and before a schema-migration command connects, Workflows compares the parsed database identities of its active DB 2 and reserved archive DB 15 with `CONTROL_REDIS_URL` and with the effective Rust data-plane URL @@ -104,6 +141,22 @@ Key concepts: retention indexes, and callbacks live in DB 2. - Workflow payloads are JSON data under explicit byte caps. Large application data should live in R2/S3/D1/KV and be referenced from workflow payloads. +- Persisted result refs are checked against the 1 MiB result limit before JSON parsing + through the shared reader used by list, get, and status. Oversized stored results are + invalid state, not caller input errors. Dispatch and restart params use the shared + JSON parser with their own 1 MiB read cap. Restart validates before creating its + pending marker or changing instance state and preserves the original JSON bytes. + Errors for missing, oversized, or malformed JSON instance-list payloads + include bounded instance-id and payload-ref context in the existing + server-side `request_complete.error_message`. Payload contents are not logged; + public 5xx responses keep the generic message. +- Instance listing reads payload refs in batches of at most 32 and releases each parsed + instance after serializing it into the bounded page. It does not retain all selected + payload JSON trees or build a second response tree. Control reads the page under the + same 8 MiB cap and a five-second backend deadline, then forwards successful JSON bytes + without re-encoding them. Oversized, unreadable, or timed-out backend pages fail as + `503 workflow_internal_dispatch_failed`; valid backend error codes retain their + existing sanitized mapping. This deadline does not change mutation retry semantics. - DB 2 keys for one instance share the `{ns:workflowKey:instanceId}` hash tag, but workflow state also uses global ready/due/retention keys. Current deployments therefore require a single non-cluster Valkey shard (`num_node_groups = 1`) rather than Redis @@ -140,7 +193,7 @@ Key families: ## Ownership / Concurrency / Failure Semantics -- Workflows are same-worker only in V2. +- Workflow bindings target definitions in the same worker. - Instances freeze the worker version/class identity they were created with. - Control fails closed on malformed active workflow entries and malformed `wf:defs` records encountered by an operation; management paths return `corrupt_meta`, while @@ -150,6 +203,37 @@ Key families: historical definitions. - Workflows lifecycle checks reject malformed referrer members instead of treating them as absent. +- Lifecycle preflight processes one bounded SSCAN page per backend call: COUNT 128 + is a hint, and pages above 512 members or 128 KiB of member bytes fail closed before + transfer. At most 20 blockers are returned; dry-run treats expired pending creates + as blockers without accumulating the whole set. Cleanup immediately applies the + existing create-token fence and prunes missing-state referrers with an atomic + absence check. Within each 20-member chunk, mutations share one pipeline; only + failed mutation slots are reread in a second batch and remain blocking while state + exists. Pipelines are not replayed after ambiguous failures. + A continuation page returns `allowed:false` with an internal cursor; it never + authorizes deletion. Control starts at zero, follows only backend cursors, renews + the held delete lock before each page, and limits a check to 16 pages / ten seconds + with five-second per-call and 64 KiB response bounds. Budget exhaustion returns + `workflow_lifecycle_check_incomplete`; retrying deletion preserves completed cleanup. + The absolute budget also bounds an in-flight page: its expiry returns the same + incomplete code and is a warning in the transport log. A per-call timeout + before the total budget expires, or a genuine backend/read failure, retains + `workflow_internal_dispatch_failed` and error-level transport diagnostics. + The limiting deadline is fixed at request setup; delayed rejection does not + turn an earlier per-call timeout into total-budget exhaustion. Equal deadlines + use the total-budget classification. + Dry-run does not prune referrers, so repeated dry-runs need not converge when + stale entries exceed the scan budget; actual deletion performs fenced cleanup. + Lock-renewal waits use the same remaining ten-second budget; this is not a deadline + for the entire public delete request or for unrelated Redis operations. + Public request cursors are not accepted as deletion authorization. + Cleanup requires the referrer set to be empty before granting permission, so a + cursor traversal interrupted by Redis failover cannot silently skip a live referrer. + If that final check still finds members without identified blockers, the response + has `allowed:false`, an empty blocker list, and no cursor. Control returns + `workflow_lifecycle_check_incomplete` and requires a fresh deletion request, rather + than reporting an active-instance conflict or restarting the scan automatically. - Scheduler only wakes workflows; workflows owns admission, fairness, shard ticks, ready/due movement, and runtime dispatch. Scheduler reads the tick response under a 64 KiB cap and requires a valid JSON object root; individual missing or unknown fields @@ -217,9 +301,18 @@ Key families: not cached and older caches are evicted under pressure. The `workflow_replay_cache_bytes` gauge reports that cross-request retained size and is published from the authoritative counters when `/_metrics` renders. Global eviction - stops cross-request retention; an in-flight controller keeps a bounded - controller-local replay working set so eviction cannot turn replay hits into fresh - claims, then releases that detached working set when dispatch ends. A new claim + stops cross-request retention; active controllers retain their detached state until + release. Retained and detached serialized bytes plus in-flight replay read reservations + share a 64 MiB budget per Runtime isolate. Identity response lengths reserve their + declared bytes; unknown/compressed responses reserve the 32 MiB reader ceiling, then + shrink to actual bytes. The reservation remains until page validation/cache admission + finishes. Unused retained caches are reclaimed first; saturation returns retryable + `workflow_backend_unavailable`, never an advisory miss or fresh claim. The last + controller releases detached state. Controller closure also cancels its pending + replay read and synchronously releases that reservation; cleanup does not depend + on an async finally running after workerd has ended the request. These are accounted + bytes, not an RSS limit. + A new claim reopens bounded paging so records committed by another isolate remain discoverable. Backend replay records are projected to the fields Runtime actually consumes before byte accounting and retention, so unconsumed response metadata cannot bypass the @@ -386,9 +479,10 @@ The step facade implements durable replay: run-token, creation-time, lease, and active-status fence. A referenced payload is resolved only while that fence remains valid, so restart cannot mix payloads from another execution generation into the page. -- V2 records a durable DAG for `step.do`. The runtime assigns ordinals synchronously in - call order, treats completed steps as the current dependency frontier, and stores the - frontier on each later step. `Promise.all([step.do(...), step.do(...)])` produces +- The workflow engine records a durable DAG for `step.do`. Runtime assigns ordinals + synchronously in call order, treats completed steps as the current dependency + frontier, and stores that frontier on each later step. + `Promise.all([step.do(...), step.do(...)])` produces sibling nodes with the same parents; a later `step.do` after the join depends on both siblings. Dependency scheduling, joins, and cancellation remain expressed by normal user-code `await` / `Promise` structure; workflows persists the resulting graph @@ -461,6 +555,15 @@ best-effort callback and records a dropped outcome; delivery is not transactiona ## Observability +Runtime publishes `wdl_workflow_replay_active_bytes`, +`wdl_workflow_replay_detached_bytes`, `wdl_workflow_replay_read_in_flight_bytes`, +`wdl_workflow_replay_working_set_bytes`, and +`wdl_workflow_replay_working_set_high_water_bytes`. Working set equals retained +cache bytes plus detached bytes plus read reservations; active bytes overlap the +retained/detached views and must not be added again. High-water lasts for the +process lifetime and is not reset by scraping. Saturation increments +`wdl_workflow_replay_cache_total{outcome="saturated"}`. + workflows follows the Rust service observability shape: JSON logs, `/_healthz`, `/_metrics`, request in-flight tracking, shutdown drain, and bounded labels. Runtime emits workflow dispatch, replay cache, payload-limit, and callback outcomes. Workflows @@ -471,6 +574,17 @@ pressure, and log workflow tick failures separately from queue/cron dispatch. ## Deployment / Rollout Notes +- Update clients that consume Workflow definition lists to follow cursors before + deploying the paginated Control endpoint. Using the companion CLI requires `1.9.0` + or later; upgrading WDL services does not upgrade installed CLI clients. + During a mixed Control/Workflows rollout, + an older Control may conservatively reject deletion on an unfinished lifecycle + page; quiesce deletion until both are updated if uninterrupted deletion is needed. +- The instance-list byte ceiling is a writer-first exception: update Workflows + before the Control worker in system-runtime. Existing readers accept shorter + pages and the unchanged cursor shape; the bounded reader must not precede the + writer that guarantees the response ceiling. This change requires no + persisted-state migration. - Cross-tier Workflow protocol changes follow the reader-before-writer procedure in the [infra rollout notes](infra.md#deployment--rollout-notes). The release changelog names the affected services. @@ -588,8 +702,7 @@ pressure, and log workflow tick failures separately from queue/cron dispatch. ## Known Constraints And Non-Goals -- V2 is not full Cloudflare Workflows compatibility. -- No cross-worker or `script_name` workflows. +- No `locationHint` placement, cross-worker or `script_name` workflows. - WDL's custom binding facade does not expose native workerd `WorkflowInstance.delete()` or `Workflow.deleteBatch()`; instance lifecycle remains owned by the documented WDL APIs and retention engine. diff --git a/docs/modules/workflows.zh.md b/docs/modules/workflows.zh.md index a1b188b1..b06e2172 100644 --- a/docs/modules/workflows.zh.md +++ b/docs/modules/workflows.zh.md @@ -8,7 +8,7 @@ Workflows 提供 Cloudflare-shaped workflow API，但由 WDL 自己的 Rust engi Workflow engine 是独立 axum 服务 `workflows`，监听 `:9120`。Runtime 通过 `runtime/workflows-client.js` 暴露 workflow binding，并通过 `runtime/dispatch/workflow-*.js` 处理 dispatch。Control 解析 workflow metadata，并拥有 deploy-time workflow definition keys。本模块文档是当前 workflows 设计参考。 -V2 这个名称用于区分当前支持 DAG 的 engine 和早期仅测试使用的 V1 engine。新环境应按 greenfield schema 3 workflows state 组织。 +新部署在专用 DB 2 中初始化 schema 3。现有 schema-2 state 需要执行离线[迁移流程](#部署--rollout-注意事项)。 workerd 提供用户代码执行环境、`WorkflowEntrypoint` class shape、module loading，以及让 runtime 在 frozen worker version 中调用 workflow class 的能力。它不提供 WDL 可复用的本地 workflow engine。WDL 在 workflows 中补齐外部 engine：DB 2 persistence、leases、ready/due scheduling、step replay、sleep、wait、event buffering、lifecycle transition、retention，以及 dispatch 回 runtime。 @@ -24,7 +24,10 @@ workerd 提供用户代码执行环境、`WorkflowEntrypoint` class shape、modu Control / CLI： - `GET /ns//workflows` 列 workflow definition，使用 `workflow.list`。 -- `GET /ns//workflows///instances` 列 instance，使用 `workflow.read`。 +- Definition list 接受 `limit`（默认 100，最多 1000）和不透明 `cursor`；每页最多处理 16 个 worker、16 MiB source snapshot 加一个有界 lookahead，并返回至多 8 MiB JSON。只保证页内排序，不保证整个 namespace 扫描的全局排序；空页或短页也必须沿 cursor 继续，直到它为 `null`。Route segment 或尚未返回完的 definition set 变化时返回 `workflow_metadata_contention`，应去掉 cursor 重新开始；这不是持久 snapshot。 +- 并发 namespace 变化可能使扫描 entry 在不同页重复；需要收集完整视图的 consumer 应按 worker/name 去重或重新扫描。 +- Cursor 输入最多 2048 个 ASCII 字节。Route discovery 使用 `HSCAN COUNT 32` 作为 hint；原生 page 超过 512 个 field/value pair 或 128 KiB field/value 总字节时，会在 transfer 前拒绝。Compact hash encoding 可能不论 COUNT 都返回整个 Hash；调高 `hash-max-listpack-entries` 等 compact-encoding threshold 因而可能让 listing fail closed。WDL 不强制这些 Valkey 配置项的取值。 +- `GET /ns//workflows///instances` 列 instance，使用 `workflow.read`。`limit` 默认 100，范围为 1-1000；每页 serialized JSON 还受 8 MiB 上限约束，包括 output/error payload 和 cursor envelope。字节预算可能使一页少于 `limit` 项；必须沿返回的 `cursor` 继续，直到它为 `null`，不能用短页判断结束。Rank cursor 计入已消费的 index entry（包括 missing/invisible instance），字节预算耗尽时从首个未返回的 instance 继续；它不提供针对并发 index 变化的 snapshot isolation。正在读取的 instance 若存在 payload-ref 错误会 fail closed，而不是静默跳过。 - `GET /ns//workflows///instances/` 返回 instance status，使用 `workflow.read`。可选查询参数只接受 camelCase：`includeSteps=true|false` 会返回 step 记录，`stepLimit=` 限制返回的 step 数。 - `POST /ns//workflows///instances//{pause,resume,restart,terminate}` 使用 `workflow.write`。 - CLI `wdl workflows list|instances|status|pause|resume|restart|terminate` 是 control API 的薄封装。 @@ -48,12 +51,19 @@ Workflows 独占 Valkey DB 2 作为 instance execution state；自定义 `WORKFL Workflows 正常启动以及 schema-migration command 连接前，都会把自身 active DB 2 与 reserved archive DB 15 的解析后 database identity，分别和 `CONTROL_REDIS_URL`、Rust data plane 的有效 URL（`DATA_REDIS_URL ?? REDIS_URL`）比较；任何重叠都会在连接前失败。使用独立 data plane 的部署必须把 canonical `DATA_REDIS_URL` 同时提供给 Workflows service，使 ownership check 能覆盖该数据库；URL credential 和无关 query setting 不改变 database identity。 +每个 worker 最多保留 1024 个 definition name，Hash field/value 总字节数最多 1 MiB；包含 materialized workflow key 的 active declaration array 也有 1 MiB JSON 上限。Deploy 在现有 WATCH 内检查累计配额后才提交；只把当前声明对应的 definition value 返回给 writer，但计数和字节统计包含 retired name。配额超限返回 `workflow_definitions_too_large`。Definition-list snapshot 在 Valkey 内先检查 cardinality 和字节数，再返回数据；单个版本的 bundle metadata 读取预算为 8 MiB，超限返回 `workflow_listing_metadata_too_large`。这是 listing 预算，不是对无关 Worker code 新增大小限制。超限存量 definition state 会 fail closed，不截断或静默兼容。 + +Definition-list snapshot 在同一个 Valkey Lua 中核验 worker 的预期 active route，并返回已经计算的总字节数用于 page accounting；Control 不再额外逐 worker 读取 route，也不重新编码已返回的字符串计数。返回 page 前仍会复核 route page fingerprint。 + 关键概念： - `workflowKey` 是 physical workflow identity。 - `(ns, worker, workflowName)` 在 redeploy 间保持稳定 workflowKey。 - Instance state、step records、payload refs、events、ready/due indexes、run leases、retention indexes 和 callbacks 存在 DB 2。 - Workflow payload 是有显式 byte cap 的 JSON data。大型 application data 应放在 R2/S3/D1/KV，再在 workflow payload 中保存引用。 +- List、get 和 status 共用 persisted result ref reader，在 JSON parse 前检查 1 MiB result limit；超限存量 result 属于 invalid state，不是当前 caller 的输入错误。Dispatch 和 restart params 复用 JSON parser，并使用自身的 1 MiB read cap。Restart 在创建 pending marker 或改变 instance state 前校验，并保留原始 JSON 字节。 +- Instance list 的 payload 缺失、超限或 JSON 非法时，会在既有服务端 `request_complete.error_message` 中包含有界的 instance-id 和 payload-ref 诊断；不记录 payload 内容，公开 5xx response 仍使用通用 message。 +- Instance list 每批最多读取 32 个 payload ref，每个 instance 在序列化进有界 page 后就释放解析后的对象，不保留全部选中 payload 的 JSON tree，也不构造第二棵 response tree。Control 使用相同的 8 MiB cap 和五秒 backend deadline 读取 page，成功时直接转发 JSON bytes，不重新编码。超限、不可读或超时的 backend page 返回 `503 workflow_internal_dispatch_failed`；合法 backend error code 沿用现有脱敏映射。该 deadline 不改变 mutation retry 语义。 - 同一个 instance 的 DB 2 key 共享 `{ns:workflowKey:instanceId}` hash tag，但 workflow state 也会使用 global ready/due/retention keys。因此当前部署要求单个非 cluster 的 Valkey 分片（`num_node_groups = 1`），而不是 Redis Cluster；为 HA 配一个 primary/replica 对是可以的，因为复制不会对 keyspace 分片，但多分片会把未加 hash tag 的 global key 拆到不同 slot 并触发 CROSSSLOT。 - Internal Durable Object alarm jobs 也存在 DB 2 的 `wf:internal:do-alarm:*` 下。它们是 Workflows-owned backend jobs，不是 tenant workflow instances，只能通过 do-runtime/workflows internal endpoints 访问。 @@ -83,10 +93,16 @@ Key families： ## Ownership / 并发 / 失败语义 -- V2 workflow 只支持 same-worker。 +- Workflow binding 只引用同一个 worker 中的 definition。 - Instance 冻结创建时的 worker version/class identity。 - Control 会对当前操作实际读到的 malformed active workflow entry 和 malformed `wf:defs` record fail closed；管理路径返回 `corrupt_meta`，deploy 在复用损坏的历史 definition 时返回 `workflow_definition_corrupt`。损坏的权威 metadata 不会被暴露为正常的 missing 或 retired workflow。正常 deploy 和单个 workflow 路径不会扫描无关的历史 definition。 - Workflows lifecycle check 会拒绝 malformed referrer member，而不是把它当成不存在。 +- Lifecycle preflight 每次 backend call 只处理一个有界 SSCAN page：COUNT 128 是 hint，超过 512 个 member 或 128 KiB member bytes 的原生 page 会在 transfer 前 fail closed。最多返回 20 个 blocker；dry-run 把 expired pending create 当作 blocker，不累计整个集合。Cleanup 应用现有 create-token fence，并用原子 absence check 清除 missing-state referrer；每个 20-member chunk 的 mutation 共用一个 pipeline，只对失败槽位再批量读取一次，state 仍存在就继续阻止删除。结果不明时不重放 pipeline。可续接的扫描页返回 `allowed:false` 与内部 cursor，不能授权删除。Control 从零开始，只跟随后端 cursor，每页前续租已有 delete lock；一次检查最多 16 页 / 十秒，每次 backend call 最多五秒、response 最多 64 KiB。预算耗尽返回 `workflow_lifecycle_check_incomplete`；再次删除会保留已完成的 cleanup 进度。公开请求中的 cursor 不作为删除授权。 +- Cleanup 授权删除前还要求 referrer set 为空，不能仅因 cursor 遍历结束就在 Redis failover 后跳过仍存活的 referrer。 +- 绝对预算同样约束在途 page：预算到期返回相同的 incomplete code，并在 transport log 中记为 warning。总预算到期前的单次请求 timeout，以及真实 backend/read failure，仍返回 `workflow_internal_dispatch_failed` 并记录 error 级 transport 诊断。Dry-run 不 prune referrer；陈旧 entry 超出扫描预算时，重复 dry-run 不保证收敛，实际删除才执行 fenced cleanup。 +- 请求建立时固定生效的 deadline；rejection 延迟送达不会把先触发的单次请求 timeout 改成总预算耗尽。两个 deadline 相同时使用总预算分类。 +- 如果最终复核仍发现 member、但没有已识别的 blocker，response 会返回 `allowed:false`、空 blocker 列表且不带 cursor。Control 返回 `workflow_lifecycle_check_incomplete`，要求重新发起删除请求，而不是报告 active-instance conflict 或自动从头重扫。 +- 续租 delete lock 的等待也受同一个十秒预算的剩余时间约束；这不是整条公开 delete request 或其他 Redis 操作的统一 deadline。 - Scheduler 只负责唤醒 workflows；admission、fairness、shard tick、ready/due movement 和 runtime dispatch 都由 workflows 负责。Scheduler 会在 64 KiB 上限内读取 tick response，并要求合法 JSON object 根节点；单个缺失或未知字段仍保持 forward-compatible，并按未报告 progress 处理。 - Scheduler 也通过同一个 `/internal/workflows/tick` endpoint 唤醒 Workflows-owned internal DO alarm jobs；scheduler 不直接读写 DO alarm state。 - Workflows 在持久化 DO alarm job 前拒绝 non-canonical alarm identity，并在 dispatch 前重新校验持久化 alarm identity。一个 Control DB Lua snapshot 会同时读取当前 storage pointer、active route、retained-version score 和 active session policy projection。当前 `restart` projection 会把 superseded alarm retarget 到 active version，即使其 scheduled version 仍 retained；后续 `preserve` projection 会覆盖尚未观察到的 restart，并让 retained alarm 继续使用 scheduled version。Malformed session policy projection、route/projection 不一致，以及 retarget 需要使用的 malformed active version 会 fail closed。其中 namespace、worker、version 校验复用 `wdl-rust-common`；do-runtime protocol grammar 与 identity helper 拥有 canonical alarm-specific field 和 aggregate 512-byte DO host-id 合同，Workflows 在持久化和 dispatch 前镜像并重新校验该合同。Runtime run dispatch 与 progress callback 在 workflows crate 内共用同一个 system-vs-user runtime endpoint selector。 @@ -94,9 +110,10 @@ Key families： - Ready token 是去重 hint；instance hash state 是权威状态。 - Runtime terminal response 是 tagged variant：`completed` 必须包含 `output` 字段，`failed` 必须包含 `error` 字段，显式 JSON `null` 仍是合法 payload。`suspended` response 只有在权威 step backend 已把同一 generation/run token 转为 `waiting` 后才会清除 run claim；格式正确但乱序的 response 会成为 fenced no-op。 - Execution commit 同时用 `generation`、`runToken`、active instance status 和未过期 run lease fence。Step commit/register 接受同一 run 的 `running` 或 `waiting` 状态，因此一个并行 sibling 进入 retry/wait 后，另一个 sibling 仍可完成；completed runtime terminal 要求 `running`，failed runtime terminal 在 run lease 仍有效时也可以关闭由非法未 await suspending step 造成的同一 run `waiting` 状态。如果 lease 已过期，workflows 只恢复 ready hint，让下一次 claim 在新 lease 下 replay。首次 run admission、过期 run requeue、lifecycle commit、`sendEvent` 与 retention cleanup 还会同时比较持久化的 `createdAtMs` 和 `generation`，避免旧 snapshot 修改同 ID 下、creation timestamp 不同的后来 incarnation；会 invalidate in-flight execution 的 lifecycle path 会在同一个 Lua commit 内 rotate `generation`。 -- Runtime replay cache 只是 advisory。DB 2 step state 是权威。同一个 runtime isolate 可以在同一 instance incarnation 的不同 run claim 间复用 terminal step record；成功 output 会保存为序列化 snapshot，并在每次 replay 时重新解码，因此租户修改返回对象不会污染后续 claim。每个 runtime isolate 的 module-level 跨请求 cache 最多保留 16 MiB serialized data；过大的 record 不进入 cache，压力下会逐出旧 cache，`workflow_replay_cache_bytes` gauge 在 `/_metrics` render 时从权威 counter 发布这部分跨请求保留量。全局逐出会停止跨请求保留；正在执行的 controller 仍可使用有界的 controller-local replay working set，避免逐出把 replay hit 变成 fresh claim，并在 dispatch 结束时释放 detached working set。Backend replay record 会先投影为 Runtime 实际消费的字段，再参与 byte accounting 和 retention，因此未消费的 response metadata 不能绕过 cache budget。新 claim 会重新开放有界分页，以发现其他 isolate 已提交的 record。 +- Runtime replay cache 只是 advisory。DB 2 step state 是权威。同一个 runtime isolate 可以在同一 instance incarnation 的不同 run claim 间复用 terminal step record；成功 output 会保存为序列化 snapshot，并在每次 replay 时重新解码，因此租户修改返回对象不会污染后续 claim。每个 runtime isolate 的 module-level 跨请求 cache 最多保留 16 MiB serialized data；过大的 record 不进入 cache，压力下会逐出旧 cache，`workflow_replay_cache_bytes` gauge 在 `/_metrics` render 时从权威 counter 发布这部分跨请求保留量。全局逐出会停止跨请求保留；active controller 保留 detached state，直到最后一个 controller 释放。Retained/detached serialized bytes 与在途 replay read reservation 共同受每个 Runtime isolate 64 MiB 预算约束。Identity response 按声明长度预留，未知长度或压缩 response 先预留 32 MiB reader ceiling，读取结束后缩到实际字节数，直到 page 校验与 cache admission 完成才释放。先回收无 active controller 的 retained cache；容量不足返回 retryable `workflow_backend_unavailable`，不转成 advisory miss 或 fresh claim。计费字节不等于 RSS。Backend replay record 会先投影为 Runtime 实际消费的字段，再参与 byte accounting 和 retention，因此未消费的 response metadata 不能绕过 cache budget。新 claim 会重新开放有界分页，以发现其他 isolate 已提交的 record。 - Runtime 可以并发发起多个 `step.do`，常见形式是 `Promise.all`；每次调用按用户代码调用顺序分配 deterministic ordinal，从当前已完成 step frontier 记录 DAG dependencies，并在 run fence 下独立 commit。Step config 是通过 workerLoader JSRPC 边界按值传递的 JSON data，callable hook 不属于该合同；Workflows 拥有 canonical config encoding 与精确的 64 KiB 上限。`step.do` callback 不能启动另一个 workflow step，即使在 callback 的 `await` 之后也不允许；并行 sibling promise 应在 run body 中、callback 代码进入 in-flight 之前创建。如果 run 在已启动 step settle 前返回，会按 invalid run 失败，所以用户代码必须 await 并发 step promise。Suspending operation（`step.sleep`、`step.sleepUntil`、`step.waitForEvent`）仍保持互斥，不能和其它 in-flight step 重叠，因为它们会 suspend 整个 workflow run。 - Completed instance 使用 success retention；failed 和 terminated instance 使用 error retention。新建 instance 的两类 retention 默认都是 8 小时，可以通过 `create({ retention: { successRetention, errorRetention } })` 覆盖。 +- Controller 关闭时会显式取消其在途 replay read，并同步、幂等地释放 read reservation；不能依赖 workerd 结束 request 后继续执行异步 finally。 - Bundled Workers types 暴露 best-effort `locationHint` create option，但 WDL 没有 regional Workflow placement plane；`create()` 与 `createBatch()` 会在 backend I/O 前拒绝自有 `locationHint` field，而不是静默丢弃。 - `Workflow.createBatch()` 每次调用最多接受 100 项；Runtime prevalidation 与 Rust admission 共享这项 pinned limit。Rust 通过一个有界 pipeline 读取去重后的 instance-state snapshot，并在各项间共享 mutation preflight；每个新 instance 仍保留独立的 create token、create 后 control-plane revalidation、cleanup 与 finalize fence。 - 单个 workflow result 的上限是 1 MiB，runtime-to-workflows backend JSON request 的上限是 2 MiB。Runtime prevalidation 和 Rust backend 共享 pinned `workflow_payload_too_large` contract。每个 instance 的 aggregate payload cap 是 16 MiB。Runtime dispatch 会在转发前限制 serialized result 与 workflows-backend request bytes；这些文档最多包含 127 层 object/array（包括平台 envelope），并拒绝 key 或 value 中的孤立 UTF-16 surrogate，与锁定的 Rust JSON parser 保持一致。Workflows 拥有 backend JSON parsing、canonical step config 和 persisted aggregate accounting。Step/event 超 cap 写入会让请求失败；runtime terminal result 超 cap 会在同一事务内把 instance 转成 failed。 @@ -129,7 +146,7 @@ Step facade 实现 durable replay： - Runtime 从头 replay 用户代码。它会 lazy fetch replay pages，也可以在进程内 advisory cache，但 DB 2 step state 始终是权威。 - Workflows step response 是 tagged variant：`claim-step` 或 `register-wait` 的 `complete` response 和 replay 的 `completed` record 必须各自拥有 `output` 字段，failed variant 必须拥有 `error` 字段。显式 JSON `null` 是合法 payload。畸形 advisory replay record 会回退到权威 `claim-step`；畸形 authoritative response 属于 result-unknown，Runtime 会重试 run，而不是伪造 null。每个新 step record 还会携带 backend-owned operation kind（`do`、`sleep`、`sleepUntil` 或 `waitForEvent`）；缺失或不匹配的 replay kind 会视为 cache miss，并回退到对应的权威 endpoint。 - Replay step record 与它引用的 payload 共用完整的 generation、run-token、creation-time、lease 和 active-status fence。只有该 fence 仍有效时才会解析引用的 payload，因此 restart 不会把另一 execution generation 的 payload 混入当前 page。 -- V2 会为 `step.do` 持久化 DAG。runtime 按同步调用顺序分配 ordinal，把已完成 step 视作当前 dependency frontier，并把 frontier 存到后续 step 上。`Promise.all([step.do(...), step.do(...)])` 会产生拥有相同 parent 的 sibling nodes；join 后再调用的 `step.do` 会依赖这两个 sibling。依赖调度、join、cancel 仍由用户代码的 `await` / `Promise` 结构表达；workflows 持久化最终 graph，不另跑一个独立 graph planner。 +- Workflows 会为 `step.do` 持久化 DAG。runtime 按同步调用顺序分配 ordinal，把已完成 step 视作当前 dependency frontier，并把 frontier 存到后续 step 上。`Promise.all([step.do(...), step.do(...)])` 会产生拥有相同 parent 的 sibling nodes；join 后再调用的 `step.do` 会依赖这两个 sibling。依赖调度、join、cancel 仍由用户代码的 `await` / `Promise` 结构表达；workflows 持久化最终 graph，不另跑一个独立 graph planner。 Fence 模型： @@ -160,10 +177,15 @@ Progress callback 是 best-effort same-worker Durable Object push。Create reque ## 可观测性 +Runtime 发布 `wdl_workflow_replay_active_bytes`、`wdl_workflow_replay_detached_bytes`、`wdl_workflow_replay_read_in_flight_bytes`、`wdl_workflow_replay_working_set_bytes` 和 `wdl_workflow_replay_working_set_high_water_bytes`。Working set 等于 retained cache、detached 与 read reservation 之和；active 是与 retained/detached 重叠的视图，不应再次相加。High-water 按进程生命周期保留，不随 scrape 重置；容量拒绝增加 `wdl_workflow_replay_cache_total{outcome="saturated"}`。 + workflows 遵循 Rust service observability shape：JSON logs、`/_healthz`、`/_metrics`、request in-flight tracking、shutdown drain 和有界 labels。Runtime 输出 workflow dispatch、replay cache、payload-limit 和 callback outcome。Workflows 通过有界 `workflow_dispatches` 输出包括 fenced no-op commit 在内的 completion outcome，并通过 `do_alarm_dispatches` 输出 internal DO alarm delivery/retry/discard/in-flight-unknown outcome。Scheduler tick 日志记录 admission 与 dispatch-pool capacity pressure，并把 workflow tick failure 与 queue/cron dispatch 分开记录。 ## 部署 / Rollout 注意事项 +- 部署分页 Control endpoint 前，先更新 definition-list client 使其沿 cursor 读取；使用 companion CLI 时，需要安装 `1.9.0` 或更高版本，升级 WDL service 不会升级已安装的 CLI client。Control 与 Workflows 混版时，旧 Control 可能对尚未完成的 lifecycle page 保守拒绝删除；若不能接受短暂拒绝，应暂停删除，直到两端更新完成。 +- Instance-list byte ceiling 是 writer-first 例外：先更新 Workflows，再更新 system-runtime 中的 Control worker。旧 reader 已接受较短 page 和不变的 cursor shape；有界 reader 不应早于保证 response ceiling 的 writer 上线。这项变更不需要 persisted-state migration。 + - 跨 tier Workflow protocol 变化遵循 [infra rollout 注意事项](infra.zh.md#部署--rollout-注意事项)中的 reader-before-writer 流程；受影响的具体 service 写入该版本 CHANGELOG。 - 必需的 runtime dispatch deadline 是一个 sender-first 例外。下述 schema-3 maintenance 会先清退全部旧 Workflows sender，再滚动 user-runtime 和 system-runtime，从而满足该约束。旧 Runtime 接受 additive field，新 Runtime 会拒绝缺少该字段的旧 sender。 - DB 2 是 workflow instance state 边界；不要从 control/runtime/scheduler 直接写 DB 2。 @@ -201,8 +223,7 @@ workflows 遵循 Rust service observability shape：JSON logs、`/_healthz`、`/ ## 已知约束和非目标 -- V2 不宣称完整 Cloudflare Workflows compatibility。 -- 不支持 cross-worker 或 `script_name` workflows。 +- 不支持 `locationHint` placement、cross-worker 或 `script_name` workflows。 - WDL 自有 binding facade 不暴露原生 workerd 的 `WorkflowInstance.delete()` 或 `Workflow.deleteBatch()`；instance lifecycle 仍由本文记录的 WDL API 和 retention engine 持有。 - 不提供平台托管的大 payload object-storage spill。 - 不使用 tenant Durable Object storage 作为 workflow backend。 diff --git a/docs/protocol-contracts.md b/docs/protocol-contracts.md index 5ecd2fc4..00cfb7db 100644 --- a/docs/protocol-contracts.md +++ b/docs/protocol-contracts.md @@ -45,6 +45,8 @@ Every protocol shape needs one owning module and one current written source: | Queue Runtime outcomes | `tests/fixtures/queue-runtime-response.json` owns the cross-language outer/inner outcome literals and required result fields consumed by Runtime dispatch and Rust Scheduler; `rust/scheduler/src/queue/delivery/outcome.rs` owns Queue decision grammar validation before destructive acknowledgement. | | Workflow definitions and instance state | `docs/modules/workflows.md`, `rust/workflows/`, runtime workflow dispatch | | Workflow Runtime requests, results, and Workflow-owned retryable backend errors | `tests/fixtures/workflow-runtime-request.json` owns the Rust-to-JavaScript run request shape, including the absolute dispatch deadline. `tests/fixtures/workflow-runtime-response.json` owns the outcome literals, required terminal payload fields, and Workflow-owned retryable error literals consumed by Runtime dispatch and Rust Workflows. Internal-auth failure uses the shared `shared/internal-auth.js` and `tests/fixtures/internal-auth-contract.json` contract. | +| Workflow payload and list limits | `tests/fixtures/workflow-limits.json` owns params/result/request byte limits, JSON grammar bounds, create-batch count, the payload-limit error code, and the aggregate instance-list response cap. Rust enforces params/result validation and the page budget; Control shares the reader cap, while Runtime consumes request/result limits. | +| Workflow deletion preflight | `tests/fixtures/workflow-lifecycle-check.json` owns the request and complete/blocked/continuation/rescan-required response variants. Rust owns bounded scan/member work; Control owns bounded continuation, response reads, and delete-lock renewal. An incomplete scan is never permission to delete; rescan-required returns the retryable incomplete error. | | Workflow schema-2 step migration | `tests/fixtures/workflow-schema2-steps.json` owns legacy record shapes and expected schema-3 kinds, consumed by the Rust offline converter and real-service migration tests. Runtime never infers missing kinds. | | Workflow step replay identity and terminal responses | `tests/fixtures/workflow-step-response.json` owns backend-written operation kinds and the required claim/replay/register-wait terminal payload fields consumed by Runtime and emitted by Rust Workflows. | | Workflow tick response | `tests/fixtures/workflow-tick-response.json` and `rust/common/src/workflow_tick.rs` own the executable shape; `rust/workflows/` and `rust/scheduler/` consume it, while `docs/modules/workflows.md` documents the surrounding behavior. | diff --git a/docs/protocol-contracts.zh.md b/docs/protocol-contracts.zh.md index 3a3d0bf9..7aee7fbd 100644 --- a/docs/protocol-contracts.zh.md +++ b/docs/protocol-contracts.zh.md @@ -36,6 +36,8 @@ | Queue Runtime outcome | `tests/fixtures/queue-runtime-response.json` 拥有 Runtime dispatch 与 Rust Scheduler 消费的跨语言 outer/inner outcome literal 和 required result field；`rust/scheduler/src/queue/delivery/outcome.rs` 负责 destructive acknowledge 前的 Queue decision grammar validation。 | | Workflow definition 和 instance state | `docs/modules/workflows.zh.md`、`rust/workflows/`、runtime workflow dispatch | | Workflow Runtime request、result 和 Workflow 自有的 retryable backend error | `tests/fixtures/workflow-runtime-request.json` 拥有 Rust 到 JavaScript 的 run request shape，包括绝对 dispatch deadline；`tests/fixtures/workflow-runtime-response.json` 拥有 Runtime dispatch 与 Rust Workflows 消费的 outcome literal、必需 terminal payload field 和 Workflow 自有 retryable error literal；internal-auth failure 使用共享的 `shared/internal-auth.js` 与 `tests/fixtures/internal-auth-contract.json` 合同。 | +| Workflow payload 与 list 上限 | `tests/fixtures/workflow-limits.json` 拥有 params/result/request byte limit、JSON grammar bound、create-batch count、payload-limit error code 和 instance-list 聚合 response cap。Rust 执行 params/result 校验和 page budget，Control 共用 reader cap，Runtime 消费 request/result limit。 | +| Workflow deletion preflight | `tests/fixtures/workflow-lifecycle-check.json` 拥有 request 与 complete/blocked/continuation/rescan-required response variant。Rust 拥有有界 scan/member work；Control 拥有有界续接、response read 和 delete-lock renewal。未完成扫描绝不是删除授权；rescan-required 返回可重试的 incomplete 错误。 | | Workflow schema-2 step migration | `tests/fixtures/workflow-schema2-steps.json` 拥有旧 record shape 与预期 schema-3 kind，由 Rust 离线转换器及真实服务迁移测试共同消费；Runtime 不推断缺失 kind。 | | Workflow step replay identity 与 terminal response | `tests/fixtures/workflow-step-response.json` 拥有 Rust Workflows 写入、Runtime 消费的 backend operation kind 与必需 claim/replay/register-wait terminal payload field。 | | Workflow tick response | `tests/fixtures/workflow-tick-response.json` 与 `rust/common/src/workflow_tick.rs` 拥有 executable shape；`rust/workflows/` 和 `rust/scheduler/` 消费该 shape，`docs/modules/workflows.zh.md` 记录外围行为。 | diff --git a/docs/redis-key-layout.md b/docs/redis-key-layout.md index 4eb2eec5..1a16b7dc 100644 --- a/docs/redis-key-layout.md +++ b/docs/redis-key-layout.md @@ -268,7 +268,9 @@ Cross-cutting constraints: is no plaintext fallback on `/runtime/load` or authenticated `/ai/resolve`. - Workflows owns DB 2 instance state. `wf:ready:cursor` is the internal ready-shard fairness cursor. Control owns only DB 0 `wf:defs:*`; other tiers must not write DB 2 - directly. + directly. Each `wf:defs::` hash retains at most 1024 names and 1 MiB + of field/value bytes, including retired definitions. Control checks quota under + WATCH before deploy commits; whole-worker delete removes the hash. - `wf:pending-version:::` is a Workflows-owned, 30-second restart blocker. Version-delete checks it with `wf:by-version`, and the successful-restart DB 2 script atomically revalidates the initial marker before replacing it with the diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index df7a9000..6200f1f3 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -133,6 +133,6 @@ Routes、platform-domain exposure、crons、queue consumers、session policy mod - Lifecycle 和 delete blocker index 在模块文档声明为权威时就是权威；不要增加绕过这些 index 的 request-path fallback scan。 - Queue main stream 不做 trim，因为 at-least-once delivery 是合同。DLQ、orphan、log-tail 这类诊断 stream 可以使用有界 approximate trim。 - Secret 和 AI credential hash value 在 steady state 下是 `WDL-ENC:` envelope。`/runtime/load` 和 authenticated `/ai/resolve` 都没有 plaintext fallback。 -- Workflows 拥有 DB 2 instance state。`wf:ready:cursor` 是内部 ready-shard 公平性 cursor。Control 只拥有 DB 0 的 `wf:defs:*`；其他 tier 不应直接写 DB 2。 +- Workflows 拥有 DB 2 instance state。`wf:ready:cursor` 是内部 ready-shard 公平性 cursor。Control 只拥有 DB 0 的 `wf:defs:*`；其他 tier 不应直接写 DB 2。每个 `wf:defs::` Hash 最多保留 1024 个 name 和 1 MiB field/value bytes，包含 retired definitions；Control 在 WATCH 内检查配额再提交 deploy，whole-worker delete 删除该 Hash。 - `wf:pending-version:::` 是 Workflows-owned、30 秒的 restart blocker。Version-delete 会将它与 `wf:by-version` 一起检查；restart 成功的 DB 2 script 会在创建持久 version referrer 前原子复验初始 marker。ZSET key 使用随写入刷新的 60 秒 TTL，确保遗留 marker key 会被物理回收。 - Workflows 还拥有 DB 2 中的 internal `wf:internal:do-alarm:*` jobs，用于 Durable Object alarm backend scheduling。do-runtime 通过 workflows HTTP API 写 alarm，而不是直接写这些 key。`wf:internal:do-alarm:ready:cursor` 是内部 ready-shard 公平性 cursor，不是租户状态。 diff --git a/docs/security.md b/docs/security.md index c4dd26ff..51e2ae40 100644 --- a/docs/security.md +++ b/docs/security.md @@ -188,7 +188,7 @@ Important role boundaries: - Delegated namespace safety assumes routine namespace-scoped writes use namespace-bound credentials. Full-plane unbound credentials can still perform namespace-scoped writes today, and those writes may leave no auth-visible namespace - fact once the active worker gate is cleared; this is an accepted V1 residual risk + fact once the active worker gate is cleared; this is an accepted residual risk until a persistent namespace fact index exists. - Platform cross-namespace visibility requires the role kind and bound namespace to match the platform-tier rule; do not replace this with route-name checks. diff --git a/docs/security.zh.md b/docs/security.zh.md index 5cb39568..027368f5 100644 --- a/docs/security.zh.md +++ b/docs/security.zh.md @@ -82,7 +82,7 @@ Auth token 是 bearer token。Auth 在 Redis 中保存 token record 和 token ha - `ns` role 绑定 tenant namespace。 - `platform` 和 `platform-observer` role 绑定 platform-tier reserved namespace。 - `token-issuer` 不绑定 namespace；除 `/whoami` self-introspection 外，它唯一 non-diagnostic action 是 `auth.delegated_token.issue`。它不能 direct issue token、list/revoke token 或访问 tenant resource。Delegated issue 返回短期 credential；namespace resource lifecycle 不属于 auth token lifecycle。 -- Delegated namespace safety 假设常规 namespace-scoped 写操作使用 namespace-bound credential。Full-plane unbound credential 目前仍能执行 namespace-scoped 写，而且 active worker gate 清空后这些写可能不留下 auth-visible namespace fact；这是 V1 接受的 residual risk，直到存在持久 namespace fact index。 +- Delegated namespace safety 假设常规 namespace-scoped 写操作使用 namespace-bound credential。Full-plane unbound credential 目前仍能执行 namespace-scoped 写，而且 active worker gate 清空后这些写可能不留下 auth-visible namespace fact；这是已接受的 residual risk，直到存在持久 namespace fact index。 - Platform 跨 namespace 可见性必须同时满足 role kind 和绑定 namespace 的 platform-tier 规则；不要用 route-name check 替代。 Control handler 不应从 URL prefix 自己推断权限。应使用 `parseControlRoute()` action classification 和 auth verify。 diff --git a/docs/source-map.md b/docs/source-map.md index 0c9a6d57..1b166c66 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -44,6 +44,7 @@ are outside this map unless they own runtime or deployable service behavior. | `control/shared.js` | Control singletons, auth wrapper, Redis publish helpers, state-bound workflow transport wiring, and shared lifecycle/delete helpers. Direct `state.*` access belongs here or in the dispatcher. | | `control/errors.js`, `control/json-body.js`, `control/optimistic.js` | Pure Control error-response and bounded JSON request-body contracts plus the strict `WatchError`/Redis-session adapter over the shared optimistic retry loop, re-exported by `control/shared.js`. | | `control/workflows-client.js` | Internal Control-to-Workflows POST transport with explicit caller-owned timeout selection; callers own endpoint-specific response interpretation. | +| `control/workflow-definitions.js` | Workflow definition quotas, bounded Redis snapshots, route-scan pages, and deploy admission accounting shared by Control listing and deploy. | | `control/lib.js` | Pure Control data-shaping: route-to-action classification, key helpers, canonical bundle `__meta__` parsing, and referrer redaction. | | `control/bundle.js` | Bundle/module normalization, compatibility metadata, vars, and emitted module manifest construction. | | `control/bindings.js` | Generic binding validation and normalization, including the singleton AI binding, plus service/platform parsing, ACL evaluation, and linker helpers. | diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index f9327982..26a244b7 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -41,6 +41,7 @@ | `control/shared.js` | Control singletons、auth wrapper、Redis publish helpers、state-bound workflow transport wiring 和共享 lifecycle/delete helpers。Direct `state.*` access 只应在这里或 dispatcher。 | | `control/errors.js`、`control/json-body.js`、`control/optimistic.js` | 由 `control/shared.js` re-export 的纯 Control error-response、bounded JSON request-body contract，以及 shared optimistic retry loop 上的 strict `WatchError`/Redis-session adapter。 | | `control/workflows-client.js` | timeout 由 caller 显式选择的 Control-to-Workflows internal POST transport；endpoint-specific response interpretation 仍由 caller 持有。 | +| `control/workflow-definitions.js` | Control listing 与 deploy 共用的 Workflow definition quota、有界 Redis snapshot、route-scan page 和 deploy admission accounting。 | | `control/lib.js` | 纯 Control data-shaping：route-to-action classification、key helpers、canonical bundle `__meta__` parsing 和 referrer redaction。 | | `control/bundle.js` | Bundle/module normalization、compatibility metadata、vars 和 emitted module manifest construction。 | | `control/bindings.js` | 通用 binding validation 和 normalization（包括 singleton AI binding），以及 service/platform parsing、ACL evaluation 和 linker helpers。 | diff --git a/docs/testing.md b/docs/testing.md index 56a425fd..a61ec00d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -152,7 +152,7 @@ should install the published CLI version pinned by the top-level `WDL_CLI_PACKAGE` value in `.github/workflows/ci.yml`: ```bash -npm install -g @wdl-dev/cli@1.8.1 +npm install -g @wdl-dev/cli@1.9.0 ``` For unpublished CLI changes, link or wrap the checkout so `wdl` is on `PATH`. diff --git a/docs/testing.zh.md b/docs/testing.zh.md index bedd25ac..bfbce6eb 100644 --- a/docs/testing.zh.md +++ b/docs/testing.zh.md @@ -113,7 +113,7 @@ CLI 子集已经包含在 `npm run test:integration` 中；只在聚焦 CLI 路 CLI integration 默认使用 `PATH` 上的 `wdl` executable。本地运行应安装 `.github/workflows/ci.yml` 顶层 `WDL_CLI_PACKAGE` pin 住的已发布 CLI 版本： ```bash -npm install -g @wdl-dev/cli@1.8.1 +npm install -g @wdl-dev/cli@1.9.0 ``` 验证未发布 CLI 变更时，优先 link 或包装 checkout，让 `wdl` 出现在 `PATH` 上。`WDL_CLI_BIN` 只保留给少数需要绕过 `PATH` 解析的聚焦 run。 diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 7afd3529..33ad7ea6 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -174,6 +174,7 @@ const controlWorker :Workerd.Worker = ( (name = "control-optimistic", esModule = embed "../control/optimistic.js"), (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), (name = "control-workflows-client", esModule = embed "../control/workflows-client.js"), + (name = "control-workflow-definitions", esModule = embed "../control/workflow-definitions.js"), (name = "shared-do-alarm-response", esModule = embed "../shared/do-alarm-response.js"), (name = "control-lib", esModule = embed "../control/lib.js"), (name = "control-d1-model", esModule = embed "../control/d1-model.js"), diff --git a/runtime/dispatch/workflow-replay-cache.js b/runtime/dispatch/workflow-replay-cache.js index 2e25c862..4e09e962 100644 --- a/runtime/dispatch/workflow-replay-cache.js +++ b/runtime/dispatch/workflow-replay-cache.js @@ -53,7 +53,15 @@ function canonicalJsonValue(value) { export const WORKFLOW_REPLAY_PAGE_SIZE = 64; export const WORKFLOW_REPLAY_CACHE_MAX_INSTANCES = 256; export const WORKFLOW_REPLAY_CACHE_MAX_BYTES = 16 * 1024 * 1024; +export const WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES = 64 * 1024 * 1024; const WORKFLOW_REPLAY_CACHE_MAX_STEPS_PER_INSTANCE = 256; + +export class WorkflowReplayCapacityError extends Error { + constructor() { + super("Workflow replay working-set capacity is exhausted"); + this.name = "WorkflowReplayCapacityError"; + } +} /** * @typedef {{ * name?: unknown, @@ -75,16 +83,36 @@ const WORKFLOW_REPLAY_CACHE_MAX_STEPS_PER_INSTANCE = 256; /** @type {Map} */ const workflowReplayCaches = new Map(); +/** @type {Set} */ +const detachedReplayCaches = new Set(); +/** @type {Set<{ reserve: (bytes: number) => void, release: () => void }>} */ +const replayReadLeases = new Set(); let workflowReplayCacheSteps = 0; let workflowReplayCacheBytes = 0; +let workflowReplayActiveBytes = 0; +let workflowReplayDetachedBytes = 0; +let workflowReplayReadBytes = 0; +let workflowReplayHighWaterBytes = 0; /** @type {WeakMap} */ let workflowReplayStepBytes = new WeakMap(); /** @lintignore data-URL unit tests import this hook from a rewritten module. */ export function _resetWorkflowReplayCacheForTest() { + for (const lease of replayReadLeases) lease.release(); + for (const cache of new Set([...workflowReplayCaches.values(), ...detachedReplayCaches])) { + cache.steps.clear(); + cache.bytes = 0; + cache.activeControllers = 0; + cache.released = true; + } workflowReplayCaches.clear(); + detachedReplayCaches.clear(); workflowReplayCacheSteps = 0; workflowReplayCacheBytes = 0; + workflowReplayActiveBytes = 0; + workflowReplayDetachedBytes = 0; + workflowReplayReadBytes = 0; + workflowReplayHighWaterBytes = 0; workflowReplayStepBytes = new WeakMap(); } @@ -92,6 +120,54 @@ export function prepareWorkflowReplayCacheMetrics() { metrics.setGauge("workflow_replay_cache_instances", {}, workflowReplayCaches.size); metrics.setGauge("workflow_replay_cache_steps", {}, workflowReplayCacheSteps); metrics.setGauge("workflow_replay_cache_bytes", {}, workflowReplayCacheBytes); + metrics.setGauge("workflow_replay_active_bytes", {}, workflowReplayActiveBytes); + metrics.setGauge("workflow_replay_detached_bytes", {}, workflowReplayDetachedBytes); + metrics.setGauge("workflow_replay_read_in_flight_bytes", {}, workflowReplayReadBytes); + metrics.setGauge("workflow_replay_working_set_bytes", {}, replayWorkingSetBytes()); + metrics.setGauge("workflow_replay_working_set_high_water_bytes", {}, workflowReplayHighWaterBytes); +} + +function replayWorkingSetBytes() { + return workflowReplayCacheBytes + workflowReplayDetachedBytes + workflowReplayReadBytes; +} + +function recordReplayHighWater() { + workflowReplayHighWaterBytes = Math.max(workflowReplayHighWaterBytes, replayWorkingSetBytes()); +} + +/** @param {number} bytes @param {WorkflowReplayCache} [keep] */ +function reserveWorkingSet(bytes, keep) { + if (replayWorkingSetBytes() + bytes <= WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES) return; + for (const [key, cache] of workflowReplayCaches) { + if (cache === keep || cache.activeControllers > 0) continue; + evictWorkflowReplayCache(key); + if (replayWorkingSetBytes() + bytes <= WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES) return; + } + recordWorkflowReplayCacheOutcome("saturated"); + throw new WorkflowReplayCapacityError(); +} + +export function createWorkflowReplayReadLease() { + let reservedBytes = 0; + let released = false; + const lease = { + /** @param {number} bytes */ + reserve(bytes) { + if (released) throw new WorkflowReplayCapacityError(); + reserveWorkingSet(Math.max(0, bytes - reservedBytes)); + workflowReplayReadBytes += bytes - reservedBytes; + reservedBytes = bytes; + recordReplayHighWater(); + }, + release() { + if (released) return; + released = true; + workflowReplayReadBytes -= reservedBytes; + replayReadLeases.delete(lease); + }, + }; + replayReadLeases.add(lease); + return lease; } /** @param {string} outcome */ @@ -149,6 +225,8 @@ function deleteReplayStep(cache, ordinal) { if (!step || !cache.steps.delete(ordinal)) return; const bytes = workflowReplayStepBytes.get(step) ?? 0; cache.bytes -= bytes; + if (cache.activeControllers > 0) workflowReplayActiveBytes -= bytes; + if (detachedReplayCaches.has(cache)) workflowReplayDetachedBytes -= bytes; if (workflowReplayCaches.get(cache.key) === cache) { workflowReplayCacheSteps -= 1; workflowReplayCacheBytes -= bytes; @@ -157,6 +235,7 @@ function deleteReplayStep(cache, ordinal) { /** @param {WorkflowReplayCache} cache */ function clearWorkflowReplayCache(cache) { + if (detachedReplayCaches.delete(cache)) workflowReplayDetachedBytes -= cache.bytes; cache.released = true; cache.steps.clear(); cache.nextOrdinal = 0; @@ -171,7 +250,12 @@ function evictWorkflowReplayCache(key) { workflowReplayCacheSteps -= cache.steps.size; workflowReplayCacheBytes -= cache.bytes; workflowReplayCaches.delete(key); - if (cache.activeControllers === 0) clearWorkflowReplayCache(cache); + if (cache.activeControllers === 0) { + clearWorkflowReplayCache(cache); + } else { + detachedReplayCaches.add(cache); + workflowReplayDetachedBytes += cache.bytes; + } } function evictOldestWorkflowReplayCache() { @@ -212,13 +296,17 @@ export function getWorkflowReplayCache(run) { /** @param {{ ns: string, workflowKey: string, instanceId: string, generation: number, createdAtMs: number, runToken: string }} run */ export function acquireWorkflowReplayCache(run) { const cache = getWorkflowReplayCache(run); + if (cache.activeControllers === 0) workflowReplayActiveBytes += cache.bytes; cache.activeControllers += 1; return cache; } /** @param {WorkflowReplayCache} cache */ export function releaseWorkflowReplayCache(cache) { - if (cache.activeControllers > 0) cache.activeControllers -= 1; + if (cache.activeControllers > 0) { + cache.activeControllers -= 1; + if (cache.activeControllers === 0) workflowReplayActiveBytes -= cache.bytes; + } if (cache.activeControllers > 0 || workflowReplayCaches.get(cache.key) === cache) return; clearWorkflowReplayCache(cache); } @@ -246,23 +334,31 @@ export function rememberWorkflowReplayStep(cache, ordinal, step) { if (storedStep.status === "completed" && typeof storedStep.outputJson !== "string") { storedStep.outputJson = JSON.stringify(step.output ?? null) ?? "null"; } - if (cache.steps.has(ordinal)) deleteReplayStep(cache, ordinal); const bytes = serializedReplayStepBytes(storedStep); if (bytes > WORKFLOW_REPLAY_CACHE_MAX_BYTES) { + deleteReplayStep(cache, ordinal); return; } + const previous = cache.steps.get(ordinal); + const oldest = !previous && cache.steps.size >= WORKFLOW_REPLAY_CACHE_MAX_STEPS_PER_INSTANCE + ? cache.steps.keys().next().value + : undefined; + const evicted = oldest === undefined ? undefined : cache.steps.get(oldest); + const replacedBytes = (previous ? workflowReplayStepBytes.get(previous) ?? 0 : 0) + + (evicted ? workflowReplayStepBytes.get(evicted) ?? 0 : 0); + reserveWorkingSet(Math.max(0, bytes - replacedBytes), cache); + deleteReplayStep(cache, ordinal); + if (oldest !== undefined) deleteReplayStep(cache, oldest); workflowReplayStepBytes.set(storedStep, bytes); cache.steps.set(ordinal, storedStep); cache.bytes += bytes; + if (cache.activeControllers > 0) workflowReplayActiveBytes += bytes; + if (detachedReplayCaches.has(cache)) workflowReplayDetachedBytes += bytes; if (countInGlobalCache) { workflowReplayCacheSteps += 1; workflowReplayCacheBytes += bytes; } - while (cache.steps.size > WORKFLOW_REPLAY_CACHE_MAX_STEPS_PER_INSTANCE) { - const oldest = cache.steps.keys().next().value; - if (oldest === undefined) break; - deleteReplayStep(cache, oldest); - } + recordReplayHighWater(); if (countInGlobalCache) { while (workflowReplayCacheBytes > WORKFLOW_REPLAY_CACHE_MAX_BYTES) { evictOldestWorkflowReplayCache(); diff --git a/runtime/dispatch/workflow-step.js b/runtime/dispatch/workflow-step.js index 26b1ac78..350821dd 100644 --- a/runtime/dispatch/workflow-step.js +++ b/runtime/dispatch/workflow-step.js @@ -11,8 +11,10 @@ import { } from "runtime-dispatch-workflow-json"; import { WORKFLOW_REPLAY_PAGE_SIZE, + WorkflowReplayCapacityError, acquireWorkflowReplayCache, canonicalJson, + createWorkflowReplayReadLease, readWorkflowReplayStepOutput, recordWorkflowReplayCacheOutcome, releaseWorkflowReplayCache, @@ -38,6 +40,7 @@ import { * @typedef {{ ordinal: number, stepName: string, nameCount: number, dependencies: number[], kind: WorkflowStepKind, config: unknown }} StepIdentity * @typedef {import("runtime-dispatch-workflow-replay-cache").WorkflowReplayStepRecord} WorkflowReplayStepRecord * @typedef {import("runtime-dispatch-workflow-replay-cache").WorkflowReplayCache} WorkflowReplayCache + * @typedef {ReturnType} WorkflowReplayReadLease */ const intrinsicObjectHasOwn = Object.hasOwn; @@ -275,16 +278,18 @@ function workflowBackendExpectedBytes(response) { * @param {string | null} requestId * @param {number} deadlineMs * @param {() => AbortSignal} deadlineSignal + * @param {WorkflowReplayReadLease} [replayLease] * @returns {Promise>} */ -async function workflowBackendCall(backend, path, body, requestId, deadlineMs, deadlineSignal) { +async function workflowBackendCall(backend, path, body, requestId, deadlineMs, deadlineSignal, replayLease) { return await workflowBackendRequest( backend, path, workflowBackendBody(path, body), requestId, deadlineMs, - deadlineSignal + deadlineSignal, + replayLease ); } @@ -295,6 +300,7 @@ async function workflowBackendCall(backend, path, body, requestId, deadlineMs, d * @param {string | null} requestId * @param {number} deadlineMs * @param {() => AbortSignal} deadlineSignal + * @param {WorkflowReplayReadLease} [replayLease] * @returns {Promise>} */ async function workflowBackendRequest( @@ -303,7 +309,8 @@ async function workflowBackendRequest( body, requestId, deadlineMs, - deadlineSignal + deadlineSignal, + replayLease ) { if (!backend || typeof backend.fetch !== "function") { throw workflowInfrastructureError("Workflow backend binding is not configured"); @@ -340,6 +347,7 @@ async function workflowBackendRequest( let parsed; try { const expectedBytes = workflowBackendExpectedBytes(response); + replayLease?.reserve(Math.min(expectedBytes ?? WORKFLOW_BACKEND_RESPONSE_MAX_BYTES, WORKFLOW_BACKEND_RESPONSE_MAX_BYTES)); const bytes = expectedBytes !== null && response.body ? await readBoundedStreamBytes( response.body, @@ -353,9 +361,11 @@ async function workflowBackendRequest( WORKFLOW_BACKEND_RESPONSE_MAX_BYTES, signal ); + replayLease?.reserve(bytes.byteLength); parsed = JSON.parse(strictUtf8Decoder.decode(bytes)); } catch (err) { void discardResponseBody(response); + if (err instanceof WorkflowReplayCapacityError) throw err; throw workflowInfrastructureError( `Workflow backend ${path} response failed with HTTP ${response.status}: ${workflowError(err).message}` ); @@ -401,15 +411,17 @@ async function workflowBackendRequest( * @param {number} ordinal * @param {string | null} requestId * @param {() => AbortSignal} deadlineSignal + * @param {WorkflowReplayReadLease} lease */ -async function fetchReplayStepPage(backend, run, cache, ordinal, requestId, deadlineSignal) { +async function fetchReplayStepPage(backend, run, cache, ordinal, requestId, deadlineSignal, lease) { while (!cache.complete && ordinal >= cache.nextOrdinal) { const startOrdinal = cache.nextOrdinal; const page = await workflowBackendCall(backend, "replay-steps", { ...workflowReplayIdentity(run), startOrdinal, limit: WORKFLOW_REPLAY_PAGE_SIZE, - }, requestId, run.dispatchDeadlineMs, deadlineSignal); + }, requestId, run.dispatchDeadlineMs, deadlineSignal, lease); + deadlineSignal().throwIfAborted(); const steps = Array.isArray(page?.steps) ? page.steps : []; const nextOrdinal = typeof page?.nextOrdinal === "number" && Number.isInteger(page.nextOrdinal) ? page.nextOrdinal @@ -425,6 +437,7 @@ async function fetchReplayStepPage(backend, run, cache, ordinal, requestId, dead } cache.nextOrdinal = Math.max(cache.nextOrdinal, nextOrdinal); if (page?.done || steps.length === 0) cache.complete = true; + lease.reserve(0); } } @@ -462,6 +475,8 @@ export function createStepController( const activeStepRecords = new Set(); /** @type {Promise | null} */ let replayFetchPromise = null; + /** @type {{ controller: AbortController, lease: WorkflowReplayReadLease } | null} */ + let replayRead = null; /** @type {AbortSignal | null} */ let backendDeadlineSignal = null; @@ -724,15 +739,20 @@ export function createStepController( * @param {Record} record */ const cacheStep = (identity, record) => { - rememberWorkflowReplayStep(replayCache, identity.ordinal, { - ordinal: identity.ordinal, - name: identity.stepName, - nameCount: identity.nameCount, - dependencies: identity.dependencies, - kind: identity.kind, - config: canonicalJson(identity.config), - ...record, - }); + try { + rememberWorkflowReplayStep(replayCache, identity.ordinal, { + ordinal: identity.ordinal, + name: identity.stepName, + nameCount: identity.nameCount, + dependencies: identity.dependencies, + kind: identity.kind, + config: canonicalJson(identity.config), + ...record, + }); + } catch (err) { + if (err instanceof WorkflowReplayCapacityError) throw workflowInfrastructureError(err.message); + throw err; + } if (identity.ordinal === replayCache.nextOrdinal) replayCache.nextOrdinal += 1; }; @@ -743,17 +763,26 @@ export function createStepController( replayCache.complete = false; } while (!replayCache.complete && targetOrdinal >= replayCache.nextOrdinal) { - replayFetchPromise ??= fetchReplayStepPage( - backend, - run, - replayCache, - targetOrdinal, - requestId, - getBackendDeadlineSignal - ) - .finally(() => { + assertRunStillOpen(); + if (replayFetchPromise === null) { + const controller = new AbortController(); + const signal = AbortSignal.any([getBackendDeadlineSignal(), controller.signal]); + const read = { controller, lease: createWorkflowReplayReadLease() }; + replayRead = read; + replayFetchPromise = fetchReplayStepPage( + backend, + run, + replayCache, + targetOrdinal, + requestId, + () => signal, + read.lease + ).finally(() => { + read.lease.release(); + if (replayRead === read) replayRead = null; replayFetchPromise = null; }); + } await replayFetchPromise; } }; @@ -763,7 +792,9 @@ export function createStepController( if (!replayCache.steps.has(identity.ordinal)) { try { await fetchReplayThrough(identity.ordinal); - } catch { + } catch (err) { + assertRunStillOpen(); + if (err instanceof WorkflowReplayCapacityError) throw workflowInfrastructureError(err.message); recordWorkflowReplayCacheOutcome("error"); return null; } @@ -1057,6 +1088,13 @@ export function createStepController( closeForRunReturn() { stepAdmissionClosed = true; runReturned = true; + if (replayRead !== null) { + const read = replayRead; + replayRead = null; + // Workerd may discard pending continuations when the request ends. + read.lease.release(); + read.controller.abort(); + } if (!replayCacheReleased) { replayCacheReleased = true; releaseWorkflowReplayCache(replayCache); diff --git a/rust/workflows/src/api.rs b/rust/workflows/src/api.rs index 0c265a70..95da219a 100644 --- a/rust/workflows/src/api.rs +++ b/rust/workflows/src/api.rs @@ -55,8 +55,8 @@ pub(crate) use lifecycle::{ use limits::{ LIFECYCLE_BLOCKER_LIMIT, MAX_CREATE_BATCH_SIZE, MAX_WORKFLOW_EVENT_BYTES, MAX_WORKFLOW_EVENT_TYPE_BYTES, MAX_WORKFLOW_INSTANCE_PAYLOAD_BYTES, - MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_PARAMS_BYTES, MAX_WORKFLOW_RESULT_BYTES, - MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES, MAX_WORKFLOW_STEP_CONFIG_BYTES, + MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES, MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_PARAMS_BYTES, + MAX_WORKFLOW_RESULT_BYTES, MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES, MAX_WORKFLOW_STEP_CONFIG_BYTES, MAX_WORKFLOW_STEP_NAME_BYTES, READY_SHARDS, WORKFLOW_PAYLOAD_TOO_LARGE_CODE, }; pub(crate) use model::{ @@ -66,12 +66,12 @@ pub(crate) use model::{ }; use payload::{ aggregate_payload_error, event_payload_json, event_type_from_value, instance_payload_limit_arg, - params_json, payload_bytes_arg, read_payload_ref, result_json, + params_json, parse_payload_json, payload_bytes_arg, read_payload_ref, result_json, }; use pending_create::{ - PENDING_CREATE_TTL_MS, PendingCreateCleanup, cleanup_created_instance, - cleanup_pending_create_identity, finalize_created_instance, is_pending_create, - pending_create_cleanup_from_state, pending_create_expired, pending_create_token, + CLEANUP_PENDING_CREATE, PENDING_CREATE_TTL_MS, PendingCreateCleanup, cleanup_created_instance, + finalize_created_instance, is_pending_create, pending_create_cleanup_from_state, + pending_create_cleanup_keys, pending_create_expired, pending_create_token, public_state_or_empty, wait_for_public_create_state, }; use pending_restart::{ diff --git a/rust/workflows/src/api/lifecycle/cleanup.rs b/rust/workflows/src/api/lifecycle/cleanup.rs index cd4bfb2f..976eb0cb 100644 --- a/rust/workflows/src/api/lifecycle/cleanup.rs +++ b/rust/workflows/src/api/lifecycle/cleanup.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use wdl_rust_common::redis_eval::StaticRedisScript; use wdl_rust_common::time::now_ms; use crate::{ @@ -7,18 +8,46 @@ use crate::{ }; use super::super::{ - LIFECYCLE_BLOCKER_LIMIT, LifecycleBlocker, LifecycleCheckRequest, LifecycleCheckResponse, - PendingCreateCleanup, active_pending_restart_blockers, cleanup_pending_create_identity, + CLEANUP_PENDING_CREATE, LIFECYCLE_BLOCKER_LIMIT, LifecycleBlocker, LifecycleCheckRequest, + LifecycleCheckResponse, PendingCreateCleanup, active_pending_restart_blockers, eval_script, is_pending_create, parse_workflow_referrer_member, pending_create_cleanup_from_state, - pending_create_expired, read_state_by_id, require_non_empty, + pending_create_cleanup_keys, pending_create_expired, require_non_empty, + workflow_referrer_member, }; +const LIFECYCLE_SCAN_COUNT: usize = 128; +const LIFECYCLE_SCAN_MAX_MEMBERS: usize = 512; +const LIFECYCLE_SCAN_MAX_BYTES: usize = 128 * 1024; + +static READ_LIFECYCLE_PAGE: StaticRedisScript = StaticRedisScript::new( + r#" +local page = redis.call("SSCAN", KEYS[1], ARGV[1], "COUNT", ARGV[2]) +if #page[2] > tonumber(ARGV[3]) then return {page[1], false} end +local bytes = 0 +for _, member in ipairs(page[2]) do + bytes = bytes + #member + if bytes > tonumber(ARGV[4]) then return {page[1], false} end +end +return page +"#, +); + +static PRUNE_MISSING_LIFECYCLE_MEMBER: StaticRedisScript = StaticRedisScript::new( + r#" +if redis.call("EXISTS", KEYS[1]) == 0 then + return redis.call("SREM", KEYS[2], ARGV[1]) +end +return 0 +"#, +); + enum LifecycleMemberState { Blocker, ExpiredPending(Box), Ignore, } +#[derive(Clone)] struct LifecycleMember { workflow_key: String, instance_id: String, @@ -29,12 +58,6 @@ struct ClassifiedLifecycleMember { state: LifecycleMemberState, } -impl LifecycleMemberState { - fn is_blocker(&self) -> bool { - matches!(self, Self::Blocker) - } -} - fn lifecycle_members_from_scan( members: impl IntoIterator, ) -> WorkflowResult> { @@ -121,14 +144,139 @@ async fn read_lifecycle_page( classify_lifecycle_page(members, states, now_ms()) } -async fn lifecycle_blocker_state( +fn lifecycle_page_response( + count: usize, + blockers: Vec, + next: u64, +) -> LifecycleCheckResponse { + let allowed = blockers.is_empty() && next == 0; + let cursor = (blockers.is_empty() && next != 0).then(|| next.to_string()); + LifecycleCheckResponse { + allowed, + count: if allowed { 0 } else { count }, + blockers, + cursor, + } +} + +fn validate_pending_scope( + req: &LifecycleCheckRequest, + member: &LifecycleMember, + pending: &PendingCreateCleanup, +) -> WorkflowResult<()> { + if pending.identity.ns != req.ns + || pending.identity.worker != req.worker + || pending.identity.workflow_key != member.workflow_key + || pending.identity.instance_id != member.instance_id + || req + .version + .as_ref() + .is_some_and(|version| version != &pending.identity.frozen_version) + { + return Err(WorkflowError::invalid_state( + "Workflow lifecycle state identity does not match its referrer", + )); + } + Ok(()) +} + +fn lifecycle_cleanup_pipeline( + req: &LifecycleCheckRequest, + key: &str, + page: &[ClassifiedLifecycleMember], +) -> WorkflowResult<(redis::Pipeline, Vec)> { + let cleanup_count = page + .iter() + .filter(|item| matches!(item.state, LifecycleMemberState::ExpiredPending(_))) + .count(); + let prune_count = page + .iter() + .filter(|item| matches!(item.state, LifecycleMemberState::Ignore)) + .count(); + let mut pipe = redis::pipe(); + let cleanup = CLEANUP_PENDING_CREATE.prepare_pipeline(&mut pipe, cleanup_count); + let prune = PRUNE_MISSING_LIFECYCLE_MEMBER.prepare_pipeline(&mut pipe, prune_count); + let mut slots = Vec::with_capacity(cleanup_count + prune_count); + for (index, classified) in page.iter().enumerate() { + let member = &classified.member; + let referrer = workflow_referrer_member(&member.workflow_key, &member.instance_id); + match &classified.state { + LifecycleMemberState::ExpiredPending(pending) => { + validate_pending_scope(req, member, pending)?; + let keys = pending_create_cleanup_keys(&pending.identity); + cleanup.append( + &mut pipe, + &keys.each_ref().map(String::as_str), + &[&referrer, &pending.token], + ); + } + LifecycleMemberState::Ignore => { + let state_key = + instance_state_key(&req.ns, &member.workflow_key, &member.instance_id); + prune.append(&mut pipe, &[&state_key, key], &[&referrer]); + } + LifecycleMemberState::Blocker => continue, + } + slots.push(index); + } + Ok((pipe, slots)) +} + +fn lifecycle_cleanup_rechecks( + page: &mut [ClassifiedLifecycleMember], + slots: &[usize], + results: Vec, +) -> WorkflowResult> { + if results.len() != slots.len() { + return Err(WorkflowError::internal_error( + "Workflow lifecycle cleanup reply count mismatch", + )); + } + let mut rechecks = Vec::new(); + for (&slot, result) in slots.iter().zip(results) { + match result { + 1 => page[slot].state = LifecycleMemberState::Ignore, + 0 => rechecks.push(slot), + _ => { + return Err(WorkflowError::internal_error( + "Workflow lifecycle cleanup result is invalid", + )); + } + } + } + Ok(rechecks) +} + +async fn cleanup_lifecycle_page( state: &AppState, - ns: &str, - workflow_key: &str, - instance_id: &str, -) -> WorkflowResult { - let existing = read_state_by_id(state, ns, workflow_key, instance_id).await?; - classify_lifecycle_member(&existing, now_ms()) + req: &LifecycleCheckRequest, + key: &str, + mut page: Vec, +) -> WorkflowResult> { + let (pipe, slots) = lifecycle_cleanup_pipeline(req, key, &page)?; + if slots.is_empty() { + return Ok(page); + } + // Each script keeps its fence; an ambiguous mixed pipeline is never replayed. + let results = state + .redis + .with_conn(async |mut conn| pipe.query_async(&mut conn).await) + .await?; + let rechecks = lifecycle_cleanup_rechecks(&mut page, &slots, results)?; + if !rechecks.is_empty() { + let members = rechecks + .iter() + .map(|&slot| page[slot].member.clone()) + .collect(); + let checked = read_lifecycle_page(state, &req.ns, members).await?; + for (slot, classified) in rechecks.into_iter().zip(checked) { + page[slot].state = match classified.state { + LifecycleMemberState::Ignore => LifecycleMemberState::Ignore, + _ => LifecycleMemberState::Blocker, + }; + } + } + Ok(page) } pub(crate) async fn check_delete_lifecycle( @@ -140,6 +288,12 @@ pub(crate) async fn check_delete_lifecycle( if let Some(version) = &req.version { require_non_empty(version, "version")?; } + let cursor = req + .cursor + .as_deref() + .unwrap_or("0") + .parse::() + .map_err(|_| WorkflowError::invalid_request("Workflow lifecycle cursor is invalid"))?; let key = match &req.version { Some(version) => by_version_key(&req.ns, &req.worker, version), None => by_worker_key(&req.ns, &req.worker), @@ -170,94 +324,61 @@ pub(crate) async fn check_delete_lifecycle( .await?; let count = referrer_count.saturating_add(pending_count); if count == 0 { - return Ok(LifecycleCheckResponse { - allowed: true, - count, - blockers: Vec::new(), - }); + return Ok(lifecycle_page_response(0, Vec::new(), 0)); } - if blockers.len() >= LIFECYCLE_BLOCKER_LIMIT { - return Ok(LifecycleCheckResponse { - allowed: false, - count, - blockers, - }); + if !blockers.is_empty() { + return Ok(lifecycle_page_response(count, blockers, 0)); } - let mut cursor = 0_u64; - let mut expired_pending = Vec::new(); - loop { - let scan_key = key.clone(); - let (next, members): (u64, Vec) = state - .redis - .with_conn(async |mut conn| { - redis::cmd("SSCAN") - .arg(scan_key) - .arg(cursor) - .arg("COUNT") - .arg(LIFECYCLE_BLOCKER_LIMIT) - .query_async(&mut conn) - .await - }) - .await?; - for chunk in members.chunks(LIFECYCLE_BLOCKER_LIMIT) { - let page = lifecycle_members_from_scan(chunk.iter().cloned())?; - for classified in read_lifecycle_page(state, &req.ns, page).await? { - match classified.state { - LifecycleMemberState::Blocker => {} - LifecycleMemberState::ExpiredPending(pending) => { - expired_pending.push(pending); - continue; - } - LifecycleMemberState::Ignore => continue, - } + let (next, members): (u64, Option>) = eval_script( + state, + &READ_LIFECYCLE_PAGE, + &[&key], + &[ + &cursor.to_string(), + &LIFECYCLE_SCAN_COUNT.to_string(), + &LIFECYCLE_SCAN_MAX_MEMBERS.to_string(), + &LIFECYCLE_SCAN_MAX_BYTES.to_string(), + ], + ) + .await?; + let members = members.ok_or_else(|| { + WorkflowError::invalid_state("Workflow lifecycle scan page exceeds its size limit") + })?; + for chunk in members.chunks(LIFECYCLE_BLOCKER_LIMIT) { + let page = lifecycle_members_from_scan(chunk.iter().cloned())?; + let mut page = read_lifecycle_page(state, &req.ns, page).await?; + if req.allow_cleanup { + page = cleanup_lifecycle_page(state, &req, &key, page).await?; + } + for classified in page { + if !matches!(classified.state, LifecycleMemberState::Ignore) { + let member = classified.member; blockers.push(LifecycleBlocker { - workflow_key: classified.member.workflow_key, - instance_id: classified.member.instance_id, + workflow_key: member.workflow_key, + instance_id: member.instance_id, }); - if blockers.len() >= LIFECYCLE_BLOCKER_LIMIT { - return Ok(LifecycleCheckResponse { - allowed: false, - count, - blockers, - }); + if blockers.len() == LIFECYCLE_BLOCKER_LIMIT { + return Ok(lifecycle_page_response(count, blockers, 0)); } } } - cursor = next; - if cursor == 0 { - break; - } } - for pending in expired_pending { - let mut is_blocker = true; - if req.allow_cleanup { - if cleanup_pending_create_identity(state, &pending.identity, &pending.token).await? { - continue; - } - is_blocker = lifecycle_blocker_state( - state, - &pending.identity.ns, - &pending.identity.workflow_key, - &pending.identity.instance_id, - ) - .await? - .is_blocker(); - } - if is_blocker { - blockers.push(LifecycleBlocker { - workflow_key: pending.identity.workflow_key, - instance_id: pending.identity.instance_id, + if req.allow_cleanup && next == 0 && blockers.is_empty() { + // A cursor traversal is not a deletion fence across reconnect/failover. + let remaining: usize = state + .redis + .with_conn(async |mut conn| redis::cmd("SCARD").arg(&key).query_async(&mut conn).await) + .await?; + if remaining > 0 { + return Ok(LifecycleCheckResponse { + allowed: false, + count: remaining, + blockers, + cursor: None, }); - if blockers.len() >= LIFECYCLE_BLOCKER_LIMIT { - break; - } } } - Ok(LifecycleCheckResponse { - allowed: blockers.is_empty(), - count: if blockers.is_empty() { 0 } else { count }, - blockers, - }) + Ok(lifecycle_page_response(count, blockers, next)) } #[cfg(test)] @@ -268,6 +389,163 @@ mod tests { use super::*; + fn cleanup_page() -> (LifecycleCheckRequest, Vec) { + let request = LifecycleCheckRequest { + ns: "demo".to_string(), + worker: "shop".to_string(), + version: Some("1".to_string()), + allow_cleanup: true, + cursor: None, + }; + let page = (0..5) + .map(|index| { + let id = format!("item-{index}"); + let state = match index { + 0 => LifecycleMemberState::Blocker, + 2 | 4 => LifecycleMemberState::Ignore, + _ => LifecycleMemberState::ExpiredPending(Box::new(PendingCreateCleanup { + identity: crate::InstanceIdentity { + ns: request.ns.clone(), + worker: request.worker.clone(), + frozen_version: "1".to_string(), + workflow_key: "flow".to_string(), + workflow_name: "orders".to_string(), + class_name: "OrderWorkflow".to_string(), + instance_id: id.clone(), + generation: "1".to_string(), + created_at_ms: "1".to_string(), + }, + token: format!("token-{index}"), + })), + }; + ClassifiedLifecycleMember { + member: LifecycleMember { + workflow_key: "flow".to_string(), + instance_id: id, + }, + state, + } + }) + .collect(); + (request, page) + } + + #[test] + fn lifecycle_cleanup_batches_mixed_actions_with_their_original_fences() { + let (request, page) = cleanup_page(); + let index_key = by_version_key("demo", "shop", "1"); + let (pipe, slots) = lifecycle_cleanup_pipeline(&request, &index_key, &page).unwrap(); + assert_eq!(slots, [1, 2, 3, 4]); + let commands: Vec> = pipe + .cmd_iter() + .map(|command| { + redis::from_redis_value( + redis::parse_redis_value(&command.get_packed_command()).unwrap(), + ) + .unwrap() + }) + .collect(); + assert_eq!( + commands.len(), + 6, + "one load per repeated script and four fenced mutations" + ); + for (slot, command) in slots.iter().zip(&commands[2..]) { + assert_eq!( + command[3], + instance_state_key("demo", "flow", &format!("item-{slot}")) + ); + let referrer = workflow_referrer_member("flow", &format!("item-{slot}")); + if slot % 2 == 1 { + assert_eq!(command[2], "9"); + assert_eq!(command[10], by_worker_key("demo", "shop")); + assert_eq!(command[11], index_key); + assert_eq!(command[12], referrer); + assert_eq!(command[13], format!("token-{slot}")); + } else { + assert_eq!(command[2], "2"); + assert_eq!(command[4], index_key); + assert_eq!(command[5], referrer); + } + } + } + + #[test] + fn lifecycle_cleanup_rechecks_only_failed_slots_and_rejects_misalignment() { + let (_, mut page) = cleanup_page(); + let slots = [1, 2, 3, 4]; + let rechecks = lifecycle_cleanup_rechecks(&mut page, &slots, vec![1, 0, 0, 1]).unwrap(); + assert_eq!(rechecks, [2, 3]); + assert!(matches!(page[0].state, LifecycleMemberState::Blocker)); + assert!(matches!(page[1].state, LifecycleMemberState::Ignore)); + assert!(matches!( + page[3].state, + LifecycleMemberState::ExpiredPending(_) + )); + assert!(matches!(page[4].state, LifecycleMemberState::Ignore)); + assert!(lifecycle_cleanup_rechecks(&mut page, &slots, vec![1]).is_err()); + assert!(lifecycle_cleanup_rechecks(&mut page, &[1], vec![2]).is_err()); + } + + #[test] + fn lifecycle_cleanup_rejects_a_foreign_pending_identity_before_execution() { + let (request, mut page) = cleanup_page(); + let LifecycleMemberState::ExpiredPending(pending) = &mut page[3].state else { + panic!("pending fixture") + }; + pending.identity.worker = "other-worker".to_string(); + let error = + lifecycle_cleanup_pipeline(&request, &by_version_key("demo", "shop", "1"), &page) + .err() + .unwrap(); + assert_eq!(error.code, "workflow_invalid_state"); + } + + #[test] + fn lifecycle_pages_match_the_control_fixture() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../../tests/fixtures/workflow-lifecycle-check.json" + )) + .unwrap(); + assert_eq!(fixture["limits"]["blockers"], LIFECYCLE_BLOCKER_LIMIT); + assert_eq!(fixture["limits"]["scanCount"], LIFECYCLE_SCAN_COUNT); + assert_eq!( + fixture["limits"]["scanMembersMax"], + LIFECYCLE_SCAN_MAX_MEMBERS + ); + assert_eq!(fixture["limits"]["scanBytesMax"], LIFECYCLE_SCAN_MAX_BYTES); + let request: LifecycleCheckRequest = + serde_json::from_value(fixture["request"].clone()).unwrap(); + assert_eq!(request.cursor.as_deref(), Some("128")); + assert!(request.allow_cleanup); + for (name, blockers, cursor) in [ + ("complete", Vec::new(), 0), + ( + "blocked", + vec![LifecycleBlocker { + workflow_key: "wf_11111111111111111111111111111111".to_string(), + instance_id: "item".to_string(), + }], + 0, + ), + ("continuation", Vec::new(), 128), + ] { + let body = + serde_json::to_value(lifecycle_page_response(123, blockers, cursor)).unwrap(); + assert_eq!(body, fixture["responses"][name]); + } + let rescan = LifecycleCheckResponse { + allowed: false, + count: 123, + blockers: Vec::new(), + cursor: None, + }; + assert_eq!( + serde_json::to_value(rescan).unwrap(), + fixture["responses"]["rescanRequired"] + ); + } + #[test] fn lifecycle_page_pipeline_preserves_valid_member_slots() { let members = lifecycle_members_from_scan(vec![ @@ -313,22 +591,31 @@ mod tests { assert_eq!(classified.len(), 2); assert_eq!(classified[0].member.workflow_key, "workflow-a"); assert_eq!(classified[0].member.instance_id, "instance-a"); - assert!(classified[0].state.is_blocker()); + assert!(matches!(classified[0].state, LifecycleMemberState::Blocker)); assert_eq!(classified[1].member.workflow_key, "workflow-b"); assert_eq!(classified[1].member.instance_id, "instance-b"); - assert!(!classified[1].state.is_blocker()); + assert!(matches!(classified[1].state, LifecycleMemberState::Ignore)); } #[test] - fn lifecycle_cleanup_failure_rechecks_blocker_state() { - let source = include_str!("cleanup.rs"); - assert!(source.contains("if req.allow_cleanup")); - assert!(source.contains( - "cleanup_pending_create_identity(state, &pending.identity, &pending.token).await?" - )); - assert!(source.contains("is_blocker = lifecycle_blocker_state(")); - assert!(source.contains(".is_blocker()")); - assert!(source.contains("blockers.push(LifecycleBlocker")); - assert!(source.contains("if blockers.len() >= LIFECYCLE_BLOCKER_LIMIT")); + fn incomplete_scan_never_authorizes_delete() { + let partial = lifecycle_page_response(1000, Vec::new(), 128); + assert!(!partial.allowed); + assert_eq!(partial.cursor.as_deref(), Some("128")); + let complete = lifecycle_page_response(1000, Vec::new(), 0); + assert!(complete.allowed); + assert_eq!(complete.count, 0); + assert!(complete.cursor.is_none()); + let blocked = lifecycle_page_response( + 1000, + vec![LifecycleBlocker { + workflow_key: "workflow".to_string(), + instance_id: "instance".to_string(), + }], + 128, + ); + assert!(!blocked.allowed); + assert_eq!(blocked.blockers.len(), 1); + assert!(blocked.cursor.is_none()); } } diff --git a/rust/workflows/src/api/lifecycle/restart.rs b/rust/workflows/src/api/lifecycle/restart.rs index 8b02787f..e39c0b2d 100644 --- a/rust/workflows/src/api/lifecycle/restart.rs +++ b/rust/workflows/src/api/lifecycle/restart.rs @@ -6,11 +6,12 @@ use crate::{ }; use super::super::{ - InstanceResponse, InstanceRouteKeys, PendingRestartMarker, WorkflowRequest, - create_pending_restart, ensure_worker_not_deleting, eval_script, identity_from_state, - instance_id, payload_bytes_arg, pending_restart_marker, read_public_state, - remove_pending_restart, request_with_active_version, validate_identity, - verify_active_workflow_current, verify_workflow_def, workflow_referrer_member, + InstanceResponse, InstanceRouteKeys, MAX_WORKFLOW_PARAMS_BYTES, PendingRestartMarker, + WorkflowRequest, create_pending_restart, ensure_worker_not_deleting, eval_script, + identity_from_state, instance_id, parse_payload_json, payload_bytes_arg, + pending_restart_marker, read_public_state, remove_pending_restart, request_with_active_version, + validate_identity, verify_active_workflow_current, verify_workflow_def, + workflow_referrer_member, }; use super::common::{ @@ -109,6 +110,7 @@ pub(crate) async fn restart_instance( }) .await? .ok_or_else(|| WorkflowError::payload_missing("Workflow params payload is missing"))?; + parse_payload_json(¶ms_json, MAX_WORKFLOW_PARAMS_BYTES)?; ensure_worker_not_deleting(state, &req.ns, &req.worker).await?; let pending_restart = pending_restart_marker(state, &req, &id); create_pending_restart(state, &pending_restart).await?; diff --git a/rust/workflows/src/api/limits.rs b/rust/workflows/src/api/limits.rs index 0b81ce75..f74cacd3 100644 --- a/rust/workflows/src/api/limits.rs +++ b/rust/workflows/src/api/limits.rs @@ -3,6 +3,7 @@ pub(crate) const MAX_WORKFLOW_PARAMS_BYTES: usize = 1024 * 1024; pub(crate) const MAX_WORKFLOW_RESULT_BYTES: usize = 1024 * 1024; pub(crate) const WORKFLOW_PAYLOAD_TOO_LARGE_CODE: &str = "workflow_payload_too_large"; pub(crate) const MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES: usize = MAX_WORKFLOW_RESULT_BYTES + 16 * 1024; +pub(crate) const MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES: usize = 8 * 1024 * 1024; pub(crate) const MAX_WORKFLOW_EVENT_BYTES: usize = 256 * 1024; pub(crate) const MAX_WORKFLOW_EVENT_TYPE_BYTES: usize = 512; pub(crate) const MAX_WORKFLOW_INSTANCE_PAYLOAD_BYTES: usize = 16 * 1024 * 1024; @@ -17,18 +18,20 @@ mod tests { use serde::Deserialize; use super::{ - MAX_CREATE_BATCH_SIZE, MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_RESULT_BYTES, - WORKFLOW_PAYLOAD_TOO_LARGE_CODE, + MAX_CREATE_BATCH_SIZE, MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES, MAX_WORKFLOW_JSON_BODY_BYTES, + MAX_WORKFLOW_PARAMS_BYTES, MAX_WORKFLOW_RESULT_BYTES, WORKFLOW_PAYLOAD_TOO_LARGE_CODE, }; #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct WorkflowLimitsContract { result_bytes_max: usize, + params_bytes_max: usize, backend_request_bytes_max: usize, json_container_depth_max: usize, reject_lone_surrogates: bool, create_batch_max: usize, + instances_response_bytes_max: usize, payload_too_large_code: String, } @@ -44,6 +47,7 @@ mod tests { .expect("workflow limits fixture"); assert_eq!(contract.result_bytes_max, MAX_WORKFLOW_RESULT_BYTES); + assert_eq!(contract.params_bytes_max, MAX_WORKFLOW_PARAMS_BYTES); assert_eq!( contract.backend_request_bytes_max, MAX_WORKFLOW_JSON_BODY_BYTES @@ -51,6 +55,10 @@ mod tests { assert_eq!(contract.json_container_depth_max, 127); assert!(contract.reject_lone_surrogates); assert_eq!(contract.create_batch_max, MAX_CREATE_BATCH_SIZE); + assert_eq!( + contract.instances_response_bytes_max, + MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES + ); assert_eq!( contract.payload_too_large_code, WORKFLOW_PAYLOAD_TOO_LARGE_CODE diff --git a/rust/workflows/src/api/model.rs b/rust/workflows/src/api/model.rs index 4f9e1498..2df645f1 100644 --- a/rust/workflows/src/api/model.rs +++ b/rust/workflows/src/api/model.rs @@ -67,7 +67,7 @@ pub(crate) struct CreateBatchResponse { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct ListInstancesResponse { - pub(crate) instances: Vec, + pub(crate) instances: Vec>, pub(crate) cursor: Option, } @@ -79,6 +79,8 @@ pub(crate) struct LifecycleCheckRequest { pub(crate) version: Option, #[serde(default)] pub(crate) allow_cleanup: bool, + #[serde(default)] + pub(crate) cursor: Option, } #[derive(Serialize)] @@ -87,6 +89,8 @@ pub(crate) struct LifecycleCheckResponse { pub(crate) allowed: bool, pub(crate) count: usize, pub(crate) blockers: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) cursor: Option, } #[derive(Serialize)] diff --git a/rust/workflows/src/api/payload.rs b/rust/workflows/src/api/payload.rs index d74cb5fe..22e85be8 100644 --- a/rust/workflows/src/api/payload.rs +++ b/rust/workflows/src/api/payload.rs @@ -97,7 +97,7 @@ pub(super) async fn read_payload_ref( .await }) .await?; - parse_payload_ref(raw, &payload_ref) + parse_payload_ref(raw, &payload_ref, MAX_WORKFLOW_RESULT_BYTES) } pub(super) fn canonical_instance_payloads_key( @@ -129,16 +129,24 @@ pub(super) fn canonical_instance_payloads_key( pub(super) fn parse_payload_ref( raw: Option, payload_ref: &str, + max_bytes: usize, ) -> WorkflowResult> { let Some(raw) = raw else { return Err(WorkflowError::payload_missing(format!( "Workflow payload reference {payload_ref} is missing" ))); }; - let parsed = serde_json::from_str(&raw).map_err(|err| { - WorkflowError::invalid_state(format!("Workflow payload is corrupt: {err}")) - })?; - Ok(Some(parsed)) + parse_payload_json(&raw, max_bytes).map(Some) +} + +pub(super) fn parse_payload_json(raw: &str, max_bytes: usize) -> WorkflowResult { + if raw.len() > max_bytes { + return Err(WorkflowError::invalid_state(format!( + "Workflow payload exceeds the {max_bytes} byte limit" + ))); + } + serde_json::from_str(raw) + .map_err(|err| WorkflowError::invalid_state(format!("Workflow payload is corrupt: {err}"))) } pub(super) fn payload_storage_key_for_ref( @@ -162,12 +170,52 @@ mod tests { #[test] fn payload_ref_missing_fails_closed() { - let err = - parse_payload_ref(None, "output").expect_err("dangling payload ref should fail closed"); + let err = parse_payload_ref(None, "output", MAX_WORKFLOW_RESULT_BYTES) + .expect_err("dangling payload ref should fail closed"); assert_eq!(err.code, "workflow_payload_missing"); assert!(err.message.contains("output")); } + #[test] + fn payload_ref_byte_limit_is_shared_by_result_and_params_readers() { + for (field, limit) in [ + ("output", MAX_WORKFLOW_RESULT_BYTES), + ("error", MAX_WORKFLOW_RESULT_BYTES), + ("params", MAX_WORKFLOW_PARAMS_BYTES), + ] { + let value = "x".repeat(limit - 2); + let exact = serde_json::to_string(&value).unwrap(); + assert_eq!(exact.len(), limit); + assert_eq!( + parse_payload_ref(Some(exact), field, limit).unwrap(), + Some(JsonValue::String(value)) + ); + let oversized = serde_json::to_string(&"x".repeat(limit - 1)).unwrap(); + let error = parse_payload_ref(Some(oversized), field, limit).unwrap_err(); + assert_eq!(error.code, "workflow_invalid_state"); + assert_eq!( + error.message, + format!("Workflow payload exceeds the {limit} byte limit") + ); + } + let error = parse_payload_ref(Some("not-json".to_string()), "output", 1).unwrap_err(); + assert_eq!(error.message, "Workflow payload exceeds the 1 byte limit"); + } + + #[test] + fn persisted_payload_json_rejects_corruption_before_reuse() { + for raw in ["", "{broken", "null trailing", r#""\ud800""#, "NaN"] { + let error = parse_payload_json(raw, MAX_WORKFLOW_PARAMS_BYTES).unwrap_err(); + assert_eq!(error.code, "workflow_invalid_state"); + assert_eq!( + parse_payload_ref(Some(raw.to_string()), "params", MAX_WORKFLOW_PARAMS_BYTES) + .unwrap_err() + .message, + error.message, + ); + } + } + #[test] fn payload_ref_requires_payloads_key() { let mut state = HashMap::new(); diff --git a/rust/workflows/src/api/pending_create.rs b/rust/workflows/src/api/pending_create.rs index 4d1239a1..4b8f8632 100644 --- a/rust/workflows/src/api/pending_create.rs +++ b/rust/workflows/src/api/pending_create.rs @@ -50,7 +50,7 @@ return 1 static FINALIZE_CREATE_INSTANCE: StaticRedisScript = StaticRedisScript::new(FINALIZE_CREATE_INSTANCE_SCRIPT); -static CLEANUP_PENDING_CREATE: StaticRedisScript = +pub(super) static CLEANUP_PENDING_CREATE: StaticRedisScript = StaticRedisScript::new(CLEANUP_PENDING_CREATE_SCRIPT); pub(super) fn pending_create_token(state: &AppState, req: &WorkflowRequest, id: &str) -> String { @@ -173,37 +173,33 @@ pub(super) async fn cleanup_pending_create_identity( identity: &InstanceIdentity, pending_create_token: &str, ) -> WorkflowResult { - let keys = InstanceRouteKeys::new(&identity.ns, &identity.workflow_key, &identity.instance_id); - let state_key = keys.state(); - let payloads_key = keys.payloads(); - let steps_key = keys.steps(); - let summaries_key = keys.step_summaries(); - let summary_index_key = keys.step_summary_index(); - let events_key = keys.events(); - let event_index_key = keys.event_type_index(); - let by_worker = by_worker_key(&identity.ns, &identity.worker); - let by_version = by_version_key(&identity.ns, &identity.worker, &identity.frozen_version); + let keys = pending_create_cleanup_keys(identity); let referrer_member = workflow_referrer_member(&identity.workflow_key, &identity.instance_id); let removed: i64 = eval_script( state, &CLEANUP_PENDING_CREATE, - &[ - &state_key, - &payloads_key, - &steps_key, - &events_key, - &summaries_key, - &summary_index_key, - &event_index_key, - &by_worker, - &by_version, - ], + &keys.each_ref().map(String::as_str), &[&referrer_member, pending_create_token], ) .await?; Ok(removed == 1) } +pub(super) fn pending_create_cleanup_keys(identity: &InstanceIdentity) -> [String; 9] { + let keys = InstanceRouteKeys::new(&identity.ns, &identity.workflow_key, &identity.instance_id); + [ + keys.state(), + keys.payloads(), + keys.steps(), + keys.events(), + keys.step_summaries(), + keys.step_summary_index(), + keys.event_type_index(), + by_worker_key(&identity.ns, &identity.worker), + by_version_key(&identity.ns, &identity.worker, &identity.frozen_version), + ] +} + pub(super) struct PendingCreateCleanup { pub(super) identity: InstanceIdentity, pub(super) token: String, diff --git a/rust/workflows/src/api/status.rs b/rust/workflows/src/api/status.rs index 84495f85..3d8bee7e 100644 --- a/rust/workflows/src/api/status.rs +++ b/rust/workflows/src/api/status.rs @@ -1,12 +1,14 @@ use std::collections::HashMap; use serde_json::{Value as JsonValue, json}; +use wdl_rust_common::text::truncate_chars; use crate::{AppState, WorkflowError, WorkflowResult, by_workflow_key, instance_state_key}; use super::payload::{canonical_instance_payloads_key, parse_payload_ref}; use super::{ - InstanceResponse, ListInstancesResponse, WorkflowRequest, instance_id, public_state_or_empty, + InstanceResponse, ListInstancesResponse, MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES, + MAX_WORKFLOW_RESULT_BYTES, WorkflowRequest, instance_id, public_state_or_empty, read_payload_ref, read_step_history, validate_identity, verify_workflow_def, workflow_step_options, }; @@ -176,14 +178,21 @@ fn prepare_list_instance( } fn apply_list_payload_reply( - instances: &mut [InstanceResponse], + instance: &mut InstanceResponse, read: &InstancePayloadRead, raw: Option, ) -> WorkflowResult<()> { - let value = parse_payload_ref(raw, &read.payload_ref)?; - let instance = instances.get_mut(read.instance_index).ok_or_else(|| { - WorkflowError::internal_error("workflow list payload response index mismatch") - })?; + let value = parse_payload_ref(raw, &read.payload_ref, MAX_WORKFLOW_RESULT_BYTES).map_err( + |mut error| { + error.message = format!( + "instance_id={:?} payload_ref={:?}: {}", + truncate_chars(&instance.id, 256), + truncate_chars(&read.payload_ref, 256), + truncate_chars(&error.message, 512), + ); + error + }, + )?; match read.field { InstancePayloadField::Output => instance.output = value, InstancePayloadField::Error => instance.error = value, @@ -191,34 +200,62 @@ fn apply_list_payload_reply( Ok(()) } -async fn read_list_payloads( +async fn read_list_payload_batch( state: &AppState, - instances: &mut [InstanceResponse], reads: &[InstancePayloadRead], -) -> WorkflowResult<()> { - for batch in reads.chunks(LIST_PAYLOAD_READ_BATCH_SIZE) { - let raw_values: Vec> = state - .redis - .with_conn(async |mut conn| { - let mut pipe = redis::pipe(); - for read in batch { - pipe.cmd("HGET") - .arg(&read.payloads_key) - .arg(&read.payload_ref); - } - pipe.query_async(&mut conn).await - }) - .await?; - if raw_values.len() != batch.len() { - return Err(WorkflowError::internal_error( - "workflow list payload reply count mismatch", +) -> WorkflowResult>> { + let raw_values: Vec> = state + .redis + .with_conn(async |mut conn| { + let mut pipe = redis::pipe(); + for read in reads { + pipe.cmd("HGET") + .arg(&read.payloads_key) + .arg(&read.payload_ref); + } + pipe.query_async(&mut conn).await + }) + .await?; + if raw_values.len() != reads.len() { + return Err(WorkflowError::internal_error( + "workflow list payload reply count mismatch", + )); + } + Ok(raw_values.into_iter()) +} + +fn list_response_overhead() -> WorkflowResult { + // Reserve the envelope and longest cursor before admitting any instance. + serde_json::to_vec(&ListInstancesResponse { + instances: Vec::new(), + cursor: Some(u64::MAX.to_string()), + }) + .map(|body| body.len()) + .map_err(|err| WorkflowError::internal_error(format!("workflow list envelope: {err}"))) +} + +fn append_list_instance( + page: &mut ListInstancesResponse, + response_bytes: &mut usize, + max_bytes: usize, + instance: &InstanceResponse, +) -> WorkflowResult { + let raw = serde_json::value::to_raw_value(instance) + .map_err(|err| WorkflowError::internal_error(format!("workflow list instance: {err}")))?; + let next_bytes = response_bytes + .saturating_add(raw.get().len()) + .saturating_add(usize::from(!page.instances.is_empty())); + if next_bytes > max_bytes { + if page.instances.is_empty() { + return Err(WorkflowError::invalid_state( + "Workflow instance exceeds the list response byte limit", )); } - for (read, raw) in batch.iter().zip(raw_values) { - apply_list_payload_reply(instances, read, raw)?; - } + return Ok(false); } - Ok(()) + *response_bytes = next_bytes; + page.instances.push(raw); + Ok(true) } pub(super) async fn read_public_state( @@ -319,30 +356,65 @@ pub(crate) async fn list_instances( }) .await? }; + if raw_states.len() != members.len() { + return Err(WorkflowError::internal_error( + "workflow list state reply count mismatch", + )); + } let mut instances = Vec::new(); let mut payload_reads = Vec::new(); - for (instance_id, raw_state) in members.iter().zip(raw_states) { + for (member_index, (instance_id, raw_state)) in members.iter().zip(raw_states).enumerate() { let existing = public_state_or_empty(state, raw_state).await?; if existing.is_empty() { continue; } let instance_index = instances.len(); - instances.push(prepare_list_instance( + let instance = prepare_list_instance( &req.ns, &req.workflow_key, instance_id, &existing, instance_index, &mut payload_reads, - )?); + )?; + instances.push((member_index, instance)); } - read_list_payloads(state, &mut instances, &payload_reads).await?; let next = cursor.saturating_add(u64::try_from(members.len()).unwrap_or(u64::MAX)); - - Ok(ListInstancesResponse { - instances, + let mut page = ListInstancesResponse { + instances: Vec::new(), cursor: has_more.then(|| next.to_string()), - }) + }; + let mut response_bytes = list_response_overhead()?; + let mut read_index = 0; + let mut raw_values = Vec::new().into_iter(); + for (instance_index, (member_index, mut instance)) in instances.into_iter().enumerate() { + while let Some(read) = payload_reads + .get(read_index) + .filter(|read| read.instance_index == instance_index) + { + if raw_values.len() == 0 { + let end = (read_index + LIST_PAYLOAD_READ_BATCH_SIZE).min(payload_reads.len()); + raw_values = + read_list_payload_batch(state, &payload_reads[read_index..end]).await?; + } + let raw = raw_values.next().ok_or_else(|| { + WorkflowError::internal_error("workflow list payload reply count mismatch") + })?; + apply_list_payload_reply(&mut instance, read, raw)?; + read_index += 1; + } + // Retain serialized rows, not every payload's parsed JSON tree. + if !append_list_instance( + &mut page, + &mut response_bytes, + MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES, + &instance, + )? { + page.cursor = Some((cursor + member_index as u64).to_string()); + break; + } + } + Ok(page) } #[cfg(test)] @@ -351,11 +423,185 @@ mod tests { use crate::api::InstanceRouteKeys; #[test] - fn list_instances_uses_bounded_workflow_index_page() { - let source = include_str!("status.rs"); - assert!(source.contains(r#"redis::cmd("ZRANGE")"#)); - assert!(!source.contains(concat!("SM", "EMBERS"))); - assert!(source.contains("members.truncate(limit)")); + fn list_options_bound_count_and_preserve_cursor() { + assert_eq!( + workflow_instances_options(&JsonValue::Null).unwrap(), + (100, 0) + ); + assert_eq!( + workflow_instances_options(&json!({"limit": 1000, "cursor": "27"})).unwrap(), + (1000, 27) + ); + for limit in [0, 1001] { + assert_eq!( + workflow_instances_options(&json!({"limit": limit})) + .unwrap_err() + .code, + "request_too_large" + ); + } + assert_eq!( + workflow_instances_options(&json!({"cursor": "invalid"})) + .unwrap_err() + .code, + "invalid_request" + ); + } + + fn list_instance(output: JsonValue, error: Option) -> InstanceResponse { + InstanceResponse { + id: "order-1".to_string(), + status: "completed".to_string(), + output: Some(output), + error, + steps: None, + } + } + + #[test] + fn list_response_budget_counts_utf8_escaping_commas_and_cursor() { + let instance = list_instance(json!({"value": "\u{4e2d}\u{1f600}\n\"\\"}), None); + let row_bytes = serde_json::to_vec(&instance).unwrap().len(); + let mut page = ListInstancesResponse { + instances: Vec::new(), + cursor: Some(u64::MAX.to_string()), + }; + let mut bytes = list_response_overhead().unwrap(); + let limit = bytes + 2 * row_bytes + 1; + assert!(append_list_instance(&mut page, &mut bytes, limit, &instance).unwrap()); + assert!(append_list_instance(&mut page, &mut bytes, limit, &instance).unwrap()); + assert!(!append_list_instance(&mut page, &mut bytes, limit, &instance).unwrap()); + assert_eq!(bytes, limit); + let serialized = serde_json::to_vec(&page).unwrap(); + assert_eq!(serialized.len(), limit); + let body: JsonValue = serde_json::from_slice(&serialized).unwrap(); + assert_eq!(body["instances"].as_array().unwrap().len(), 2); + assert_eq!( + body["instances"][0], + serde_json::to_value(&instance).unwrap() + ); + assert!(body["instances"][0].get("steps").is_none()); + + let mut shorter = ListInstancesResponse { + instances: Vec::new(), + cursor: None, + }; + let mut bytes = list_response_overhead().unwrap(); + assert!(append_list_instance(&mut shorter, &mut bytes, limit - 1, &instance).unwrap()); + assert!(!append_list_instance(&mut shorter, &mut bytes, limit - 1, &instance).unwrap()); + } + + #[test] + fn list_response_byte_limit_shortens_a_legal_large_payload_page() { + let instance = list_instance( + json!("x".repeat(MAX_WORKFLOW_RESULT_BYTES - 2)), + Some(JsonValue::Null), + ); + let mut page = ListInstancesResponse { + instances: Vec::new(), + cursor: None, + }; + let mut bytes = list_response_overhead().unwrap(); + let mut admitted = 0; + for _ in 0..MAX_INSTANCES_LIMIT { + if !append_list_instance( + &mut page, + &mut bytes, + MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES, + &instance, + ) + .unwrap() + { + break; + } + admitted += 1; + } + assert_eq!(admitted, 7); + assert!(serde_json::to_vec(&page).unwrap().len() <= MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES); + assert_eq!(page.instances.len(), admitted); + } + + #[test] + fn list_response_rejects_an_unpageable_instance_instead_of_stalling_cursor() { + let instance = list_instance(JsonValue::Null, None); + let mut page = ListInstancesResponse { + instances: Vec::new(), + cursor: None, + }; + let mut bytes = list_response_overhead().unwrap(); + let limit = bytes + serde_json::to_vec(&instance).unwrap().len() - 1; + let error = append_list_instance(&mut page, &mut bytes, limit, &instance).unwrap_err(); + assert_eq!(error.code, "workflow_invalid_state"); + assert!(page.instances.is_empty()); + } + + #[test] + fn list_payload_rejects_oversized_persisted_json_before_parsing() { + let mut instance = list_instance(JsonValue::Null, None); + let read = InstancePayloadRead { + instance_index: 0, + field: InstancePayloadField::Output, + payloads_key: "payloads".to_string(), + payload_ref: "output".to_string(), + }; + let error = apply_list_payload_reply( + &mut instance, + &read, + Some("x".repeat(MAX_WORKFLOW_RESULT_BYTES + 1)), + ) + .unwrap_err(); + assert_eq!(error.code, "workflow_invalid_state"); + assert_eq!( + error.message, + format!( + "instance_id=\"order-1\" payload_ref=\"output\": Workflow payload exceeds the {MAX_WORKFLOW_RESULT_BYTES} byte limit" + ) + ); + } + + #[tokio::test] + async fn list_payload_diagnostics_are_bounded_and_stay_out_of_the_response() { + use axum::response::IntoResponse; + + let mut instance = list_instance(JsonValue::Null, None); + instance.id = "\u{4e2d}".repeat(300); + let read = InstancePayloadRead { + instance_index: 0, + field: InstancePayloadField::Error, + payloads_key: "payloads".to_string(), + payload_ref: "r".repeat(300), + }; + let error = apply_list_payload_reply( + &mut instance, + &read, + Some("{\"private-payload\":".to_string()), + ) + .unwrap_err(); + assert_eq!(error.code, "workflow_invalid_state"); + assert!( + error + .message + .contains(&format!("instance_id={:?}", "\u{4e2d}".repeat(256))) + ); + assert!( + error + .message + .contains(&format!("payload_ref={:?}", "r".repeat(256))) + ); + assert!(error.message.chars().count() < 1100); + assert!(!error.message.contains("private-payload")); + let response = error.into_response(); + assert_eq!( + response.status(), + axum::http::StatusCode::INTERNAL_SERVER_ERROR + ); + let body = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap(), + json!({"error": "workflow_invalid_state", "message": "Workflow service request failed"}) + ); } #[test] @@ -397,7 +643,7 @@ mod tests { let mut reads = Vec::new(); let response = prepare_list_instance("demo", "wf_test", "inst-1", &state, 0, &mut reads) .expect("valid list instance"); - let mut instances = vec![response]; + let mut instances = [response]; assert_eq!(reads.len(), 2); assert!(matches!(reads[0].field, InstancePayloadField::Output)); @@ -406,13 +652,13 @@ mod tests { assert_eq!(reads[1].payload_ref, "shared-ref"); apply_list_payload_reply( - &mut instances, + &mut instances[0], &reads[0], Some(r#"{"kind":"output"}"#.to_string()), ) .expect("output payload"); apply_list_payload_reply( - &mut instances, + &mut instances[0], &reads[1], Some(r#"{"kind":"error"}"#.to_string()), ) @@ -437,10 +683,14 @@ mod tests { let mut reads = Vec::new(); let response = prepare_list_instance("demo", "wf_test", "inst-1", &state, 0, &mut reads) .expect("valid list instance"); - let mut instances = vec![response]; + let mut instances = [response]; - let err = apply_list_payload_reply(&mut instances, &reads[0], None) + let err = apply_list_payload_reply(&mut instances[0], &reads[0], None) .expect_err("missing list payload must fail closed"); assert_eq!(err.code, "workflow_payload_missing"); + assert!( + err.message + .contains("instance_id=\"inst-1\" payload_ref=\"missing-ref\"") + ); } } diff --git a/rust/workflows/src/api/tick.rs b/rust/workflows/src/api/tick.rs index 2efd9f60..2e7d43e3 100644 --- a/rust/workflows/src/api/tick.rs +++ b/rust/workflows/src/api/tick.rs @@ -12,10 +12,10 @@ use crate::{ use super::do_alarms::DoAlarmAdmissionResult; use super::payload::{parse_payload_ref, payload_storage_key_for_ref}; use super::{ - InstanceRouteKeys, ReadyAdmissionConfig, ReadyAdmissionOutcome, RunClaim, - admit_ready_do_alarms, admit_ready_members, claim_run, eval_script, identity_from_state, - log_instance_event, parse_ready_token, release_run_claim, requeue_expired_run_claim, - spawn_progress_from_identity, + InstanceRouteKeys, MAX_WORKFLOW_PARAMS_BYTES, ReadyAdmissionConfig, ReadyAdmissionOutcome, + RunClaim, admit_ready_do_alarms, admit_ready_members, claim_run, eval_script, + identity_from_state, log_instance_event, parse_ready_token, release_run_claim, + requeue_expired_run_claim, spawn_progress_from_identity, }; mod dispatch; @@ -107,7 +107,10 @@ fn params_from_ready_snapshot( "Workflow payload storage key is invalid", )); } - Ok(parse_payload_ref(raw_params, params_ref)?.unwrap_or(JsonValue::Null)) + Ok( + parse_payload_ref(raw_params, params_ref, MAX_WORKFLOW_PARAMS_BYTES)? + .unwrap_or(JsonValue::Null), + ) } async fn read_ready_state_and_params( diff --git a/rust/workflows/src/server.rs b/rust/workflows/src/server.rs index 54572789..74ef28a5 100644 --- a/rust/workflows/src/server.rs +++ b/rust/workflows/src/server.rs @@ -4,15 +4,15 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use crate::{ - AppState, DispatchSemaphores, LogLevel, Metrics, Redis, SERVICE, ShutdownState, WorkflowError, - WorkflowResult, check_delete_lifecycle, claim_step, commit_step_error, commit_step_success, - config_from_env, create_batch, create_instance, ensure_schema_migration_complete, - ensure_workflows_schema, get_instance, list_instances, log, pause_instance, - read_do_alarm_cleanup_request, read_do_alarm_delete_request, read_do_alarm_set_request, - read_lifecycle_check_request, read_replay_step_page, read_workflow_replay_request, - read_workflow_request, read_workflow_step_request, register_sleep, register_wait, - restart_instance, resume_instance, send_event, status_instance, terminate_instance, - tick_workflows, workflow_error_fields, + AppState, DispatchSemaphores, ListInstancesResponse, LogLevel, Metrics, Redis, SERVICE, + ShutdownState, WorkflowError, WorkflowResult, check_delete_lifecycle, claim_step, + commit_step_error, commit_step_success, config_from_env, create_batch, create_instance, + ensure_schema_migration_complete, ensure_workflows_schema, get_instance, list_instances, log, + pause_instance, read_do_alarm_cleanup_request, read_do_alarm_delete_request, + read_do_alarm_set_request, read_lifecycle_check_request, read_replay_step_page, + read_workflow_replay_request, read_workflow_request, read_workflow_step_request, + register_sleep, register_wait, restart_instance, resume_instance, send_event, status_instance, + terminate_instance, tick_workflows, workflow_error_fields, }; use axum::body::Body; use axum::extract::State; @@ -105,10 +105,10 @@ async fn get_handler(State(state): State, body: Body) -> WorkflowResul async fn instances_handler( State(state): State, body: Body, -) -> WorkflowResult> { +) -> WorkflowResult> { let req = read_workflow_request(body).await?; let response = list_instances(&state, req).await?; - workflow_json(response, "workflow instances response serializes") + Ok(Json(response)) } async fn status_handler( diff --git a/scripts/integration-test-plan.js b/scripts/integration-test-plan.js index c928e266..1e9c1b27 100644 --- a/scripts/integration-test-plan.js +++ b/scripts/integration-test-plan.js @@ -5,69 +5,69 @@ const ROOT = path.resolve(import.meta.dirname, ".."); export const DEFAULT_INTEGRATION_DURATIONS_FILE = path.join(ROOT, ".integration-test-durations.json"); // Update when integration runs reveal new slow tests; ordering affects -// scheduling latency, not correctness. Last calibrated 2026-09-02 against the -// complete 58-file .integration-test-durations.json run (descending +// scheduling latency, not correctness. Last calibrated 2026-09-12 (UTC) against +// the complete 58-file .integration-test-durations.json run (descending // durationMs). The recorded duration file, when present, takes precedence over // this fallback list. export const SLOW_FIRST_FILES = [ "durable-objects-ownership.test.js", "d1-storage-shared-localdisk.test.js", "d1-ownership-multi-runtime.test.js", - "durable-objects-eviction.test.js", + "workflows-schema-migration.test.js", "durable-objects-alarms.test.js", + "durable-objects-eviction.test.js", "gateway-websocket.test.js", + "cron-triggers.test.js", "log-tail.test.js", "durable-objects-websocket.test.js", "workflows-runtime-scheduler.test.js", "queues-retry-and-delay.test.js", - "cron-triggers.test.js", - "workflows-schema-migration.test.js", - "queues-orphan-and-control.test.js", - "queues-delivery.test.js", "ai-binding.test.js", - "d1-read-cache.test.js", "workflows-runtime-core.test.js", - "queues-batch-and-isolation.test.js", - "delete-api.test.js", - "secrets.test.js", + "queues-delivery.test.js", + "queues-orphan-and-control.test.js", + "d1-read-cache.test.js", "workflows-runtime-retention.test.js", + "delete-api.test.js", + "queues-batch-and-isolation.test.js", "redis-conformance.test.js", "durable-objects-storage.test.js", "d1-binding.test.js", + "secrets.test.js", "kv-binding.test.js", + "cli-smoke.test.js", "auth-platform.test.js", "admin-api.test.js", - "gateway.test.js", "workflows-runtime-pausing.test.js", - "scheduler-shutdown-drain.test.js", + "gateway.test.js", "auth-worker.test.js", - "cli-smoke.test.js", - "d1-storage-localdisk.test.js", "service-bindings-rpc.test.js", - "d1-lifecycle.test.js", + "scheduler-shutdown-drain.test.js", + "d1-storage-localdisk.test.js", "r2-cli-binding.test.js", - "s3-cleanup.test.js", + "observability.test.js", "http-features.test.js", + "s3-cleanup.test.js", "routing-gateway.test.js", - "cli-multi-env.test.js", - "platform-bindings.test.js", - "observability.test.js", - "queue-native-dispatch.test.js", + "workflows-metadata.test.js", + "d1-lifecycle.test.js", "network-boundary.test.js", "delete-indexes.test.js", + "platform-bindings.test.js", "runtime-eviction.test.js", - "durable-objects-core.test.js", - "pages-assets-demo.test.js", - "worker-modules.test.js", - "routing-admin.test.js", - "system-pool-auth.test.js", + "queue-native-dispatch.test.js", "assets-binding.test.js", + "cli-multi-env.test.js", "service-bindings.test.js", + "durable-objects-core.test.js", "cli-workers-dev-optout.test.js", "route-demo.test.js", + "pages-assets-demo.test.js", + "system-pool-auth.test.js", + "worker-modules.test.js", + "routing-admin.test.js", "workflows-durable-objects.test.js", "workflows-service.test.js", - "workflows-metadata.test.js", ]; export const CLI_INTEGRATION_MARKER = "@wdl-cli-integration"; diff --git a/terraform/README.md b/terraform/README.md index 62b5a608..f72aa387 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -226,6 +226,16 @@ private plus public mesh reach. ## Capacity +All seven service families expose desired-count inputs, including +`system_runtime_desired_count` (default `1`). Operators can set every count to `0` +for full-stop maintenance. Wait for old tasks to stop, apply and converge the +selected image revisions at zero capacity, then restore capacity separately in +the release's required startup order. + +Setting only `system_runtime_desired_count = 0` also stops Control/Auth and +`__system__` Workers. Admin APIs, deployment, and system-targeted dispatch are +then unavailable even if other pools continue serving existing tenant workloads. + The ECS cluster enables both `FARGATE` and `FARGATE_SPOT` capacity providers: - gateway, user-runtime, and system-runtime keep `base = 1` on on-demand Fargate and diff --git a/terraform/main.tf b/terraform/main.tf index ee1d025e..faa63454 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -40,28 +40,29 @@ module "compute" { secret_envelope_secret_arn = module.data.secret_envelope_secret_arn internal_auth_previous_token_secret_arn = var.internal_auth_previous_token_secret_arn - gateway_desired_count = var.gateway_desired_count - runtime_desired_count = var.runtime_desired_count - d1_runtime_desired_count = var.d1_runtime_desired_count - do_runtime_desired_count = var.do_runtime_desired_count - do_prevent_eviction = var.do_prevent_eviction - d1_test_hooks_enabled = var.d1_test_hooks_enabled - scheduler_desired_count = var.scheduler_desired_count - workflows_desired_count = var.workflows_desired_count - gateway_cpu = var.gateway_cpu - gateway_memory = var.gateway_memory - system_runtime_cpu = var.system_runtime_cpu - system_runtime_memory = var.system_runtime_memory - runtime_cpu = var.runtime_cpu - runtime_memory = var.runtime_memory - d1_runtime_container_memory = var.d1_runtime_container_memory - do_runtime_container_memory = var.do_runtime_container_memory - scheduler_cpu = var.scheduler_cpu - scheduler_memory = var.scheduler_memory - workflows_cpu = var.workflows_cpu - workflows_memory = var.workflows_memory - spot_weight = var.spot_weight - od_weight = var.od_weight + gateway_desired_count = var.gateway_desired_count + runtime_desired_count = var.runtime_desired_count + system_runtime_desired_count = var.system_runtime_desired_count + d1_runtime_desired_count = var.d1_runtime_desired_count + do_runtime_desired_count = var.do_runtime_desired_count + do_prevent_eviction = var.do_prevent_eviction + d1_test_hooks_enabled = var.d1_test_hooks_enabled + scheduler_desired_count = var.scheduler_desired_count + workflows_desired_count = var.workflows_desired_count + gateway_cpu = var.gateway_cpu + gateway_memory = var.gateway_memory + system_runtime_cpu = var.system_runtime_cpu + system_runtime_memory = var.system_runtime_memory + runtime_cpu = var.runtime_cpu + runtime_memory = var.runtime_memory + d1_runtime_container_memory = var.d1_runtime_container_memory + do_runtime_container_memory = var.do_runtime_container_memory + scheduler_cpu = var.scheduler_cpu + scheduler_memory = var.scheduler_memory + workflows_cpu = var.workflows_cpu + workflows_memory = var.workflows_memory + spot_weight = var.spot_weight + od_weight = var.od_weight log_level = var.log_level log_retention_days = var.log_retention_days diff --git a/terraform/modules/compute/system_runtime_service.tf b/terraform/modules/compute/system_runtime_service.tf index fc495632..63ac1ee8 100644 --- a/terraform/modules/compute/system_runtime_service.tf +++ b/terraform/modules/compute/system_runtime_service.tf @@ -129,9 +129,9 @@ module "system_runtime_service" { name = "${var.name}-system-runtime" cluster_id = aws_ecs_cluster.this.id task_definition_arn = aws_ecs_task_definition.system_runtime.arn - # Day-one single replica. Control is stateless (Redis is the truth), - # so scaling out is purely a throughput choice, not correctness. - desired_count = 1 + # Defaults to one replica; operators may adjust capacity or use zero for maintenance. + # Control is stateless (Redis is the truth), so scaling out is a throughput choice. + desired_count = var.system_runtime_desired_count enable_execute_command = true deployment = local.start_before_stop_deployment diff --git a/terraform/modules/compute/variables.tf b/terraform/modules/compute/variables.tf index 1e176fc0..36b9e1d9 100644 --- a/terraform/modules/compute/variables.tf +++ b/terraform/modules/compute/variables.tf @@ -41,6 +41,7 @@ variable "internal_auth_previous_token_secret_arn" { variable "gateway_desired_count" { type = number } variable "runtime_desired_count" { type = number } +variable "system_runtime_desired_count" { type = number } variable "d1_runtime_desired_count" { type = number } variable "do_runtime_desired_count" { type = number } variable "do_prevent_eviction" { type = bool } diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index 294a5be8..c47d3510 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -23,6 +23,10 @@ alb_security_group_id = "sg-xxxxxxxxxxxxxxxxx" d1_test_hooks_enabled = false do_prevent_eviction = true +# Defaults to one; set to zero alongside the other service counts for full-stop +# maintenance, then restore it before reopening control-plane traffic. +system_runtime_desired_count = 1 + # Stateless gateway/user-runtime/system-runtime keep at least one on-demand # Fargate task and place overflow according to these weights. Stateful runtimes, # scheduler, and workflows stay on on-demand Fargate. diff --git a/terraform/variables.tf b/terraform/variables.tf index 644960b0..c26e0e27 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -149,6 +149,11 @@ variable "runtime_desired_count" { default = 2 } +variable "system_runtime_desired_count" { + type = number + default = 1 +} + variable "d1_runtime_desired_count" { type = number default = 2 diff --git a/test-workers/workflow-replay-capacity/src/index.js b/test-workers/workflow-replay-capacity/src/index.js new file mode 100644 index 00000000..5d117552 --- /dev/null +++ b/test-workers/workflow-replay-capacity/src/index.js @@ -0,0 +1,28 @@ +import { WorkflowEntrypoint } from "cloudflare:workers"; + +const callbacks = new Map(); +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +export class ReplayCapacityWorkflow extends WorkflowEntrypoint { + async run(event, step) { + const id = event.payload.id; + for (let index = 0; index < 6; index += 1) { + await step.do(`payload-${index}`, async () => { + callbacks.set(id, (callbacks.get(id) ?? 0) + 1); + return "x".repeat(768 * 1024); + }); + } + while (await this.env.STATE.get("release") !== "1") await delay(50); + return { id, callbacks: callbacks.get(id) ?? 0 }; + } +} + +export default { + async fetch(request, env) { + if (new URL(request.url).pathname.endsWith("/release")) { + await env.STATE.put("release", "1"); + return Response.json({ released: true }); + } + return Response.json({ callbacks: Object.fromEntries(callbacks) }); + }, +}; diff --git a/test-workers/workflow-replay-lifetime/src/index.js b/test-workers/workflow-replay-lifetime/src/index.js new file mode 100644 index 00000000..ad6c7580 --- /dev/null +++ b/test-workers/workflow-replay-lifetime/src/index.js @@ -0,0 +1,42 @@ +import { createStepController } from "./workflow-step.js"; +import { prepareWorkflowReplayCacheMetrics } from "./workflow-replay-cache.js"; +import { gauges } from "./metrics.js"; + +let invocations = 0; +let cancellations = 0; + +function state() { + prepareWorkflowReplayCacheMetrics(); + return { invocations, cancellations, bytes: gauges.get("workflow_replay_read_in_flight_bytes") }; +} + +export default { + async fetch(request, env) { + if (new URL(request.url).pathname.endsWith("/state")) return Response.json(state()); + invocations += 1; + const started = Promise.withResolvers(); + const controller = createStepController({ + ns: "probe", worker: "probe", frozenVersion: "v1", workflowName: "probe", + workflowKey: "wf_probe", className: "Probe", instanceId: String(invocations), + generation: 1, runToken: "probe", createdAtMs: 1, dispatchDeadlineMs: Date.now() + 2_000, + }, { + async fetch() { + return new Response(new ReadableStream({ + pull() { started.resolve(undefined); }, + cancel() { cancellations += 1; }, + }, { highWaterMark: 0 }), { headers: { "content-length": String(1024 * 1024) } }); + }, + }); + let before; + let rootError; + try { + await env.ROOT.run(controller.facade, () => started.promise); + } catch (error) { + rootError = error.message; + before = state(); + } finally { + controller.closeForRunReturn(); + } + return Response.json({ rootError, before, after: state() }, { status: 503 }); + }, +}; diff --git a/test-workers/workflow-replay-lifetime/src/metrics.js b/test-workers/workflow-replay-lifetime/src/metrics.js new file mode 100644 index 00000000..5ee74343 --- /dev/null +++ b/test-workers/workflow-replay-lifetime/src/metrics.js @@ -0,0 +1,5 @@ +export const gauges = new Map(); +export const metrics = { + increment() {}, + setGauge(name, _labels, value) { gauges.set(name, value); }, +}; diff --git a/test-workers/workflow-replay-lifetime/src/root.js b/test-workers/workflow-replay-lifetime/src/root.js new file mode 100644 index 00000000..3cbcf258 --- /dev/null +++ b/test-workers/workflow-replay-lifetime/src/root.js @@ -0,0 +1,10 @@ +import { WorkerEntrypoint } from "cloudflare:workers"; + +export default class extends WorkerEntrypoint { + async run(step, started) { + step.do("pending", async () => "unexpected callback").catch(() => {}); + await started(); + throw new Error("root escaped"); + } + async fetch() { return Response.json({ ready: true }); } +} diff --git a/tests/fixtures/workflow-lifecycle-check.json b/tests/fixtures/workflow-lifecycle-check.json new file mode 100644 index 00000000..a93dad1f --- /dev/null +++ b/tests/fixtures/workflow-lifecycle-check.json @@ -0,0 +1,28 @@ +{ + "limits": { + "blockers": 20, + "scanCount": 128, + "scanMembersMax": 512, + "scanBytesMax": 131072, + "responseBytesMax": 65536, + "controlPagesMax": 16, + "controlTimeoutMs": 10000 + }, + "request": { + "ns": "demo", + "worker": "api", + "version": "v2", + "allowCleanup": true, + "cursor": "128" + }, + "responses": { + "complete": { "allowed": true, "count": 0, "blockers": [] }, + "blocked": { + "allowed": false, + "count": 123, + "blockers": [{ "workflowKey": "wf_11111111111111111111111111111111", "instanceId": "item" }] + }, + "continuation": { "allowed": false, "count": 123, "blockers": [], "cursor": "128" }, + "rescanRequired": { "allowed": false, "count": 123, "blockers": [] } + } +} diff --git a/tests/fixtures/workflow-limits.json b/tests/fixtures/workflow-limits.json index 28137434..8727099a 100644 --- a/tests/fixtures/workflow-limits.json +++ b/tests/fixtures/workflow-limits.json @@ -1,8 +1,10 @@ { "resultBytesMax": 1048576, + "paramsBytesMax": 1048576, "backendRequestBytesMax": 2097152, "jsonContainerDepthMax": 127, "rejectLoneSurrogates": true, "createBatchMax": 100, + "instancesResponseBytesMax": 8388608, "payloadTooLargeCode": "workflow_payload_too_large" } diff --git a/tests/helpers/control-shared-stub.js b/tests/helpers/control-shared-stub.js index bc44a6cc..75c5e02b 100644 --- a/tests/helpers/control-shared-stub.js +++ b/tests/helpers/control-shared-stub.js @@ -23,6 +23,7 @@ const { const CONTROL_SHARED_BASE = ` import { jsonError, jsonResponse, sanitizeJsonErrorDetails } from ${JSON.stringify(SHARED_RESPOND_URL)}; import { createPostWorkflowsInternal } from ${JSON.stringify(CONTROL_WORKFLOWS_CLIENT_URL)}; +export { WORKFLOWS_INTERNAL_TIMEOUT_MS, readWorkflowInstancesResponse } from ${JSON.stringify(CONTROL_WORKFLOWS_CLIENT_URL)}; import { ControlAbort, codedErrorLogFields, codedErrorResponse, controlAbortResponse, secretEnvelopeErrorResponse } from ${JSON.stringify(CONTROL_ERRORS_URL)}; import { runOptimistic, withOptimisticRetries } from ${JSON.stringify(CONTROL_OPTIMISTIC_URL)}; import { readJsonBody } from ${JSON.stringify(CONTROL_JSON_BODY_URL)}; diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index a8fbc66b..a3521dae 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -9,6 +9,7 @@ import { repositoryFileUrl, } from "./load-shared-module.js"; import { compileSharedAuthRoles } from "./load-auth-roles.js"; +import { workflowDefinitionsUrl } from "./workflow-definitions.js"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // Resolve croner against the root package so tests pick up the hoisted @@ -83,6 +84,7 @@ export async function compileControlGraph(opts = {}) { "base64.js": SHARED_BASE64_URL, "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, "control-lib": libUrl, + "control-workflow-definitions": workflowDefinitionsUrl, }), [/from "control-bindings"/g, `from ${JSON.stringify(bindingsUrl)}`], [/from "wdl-package-json-source"/g, `from ${JSON.stringify(packageJsonSourceUrl)}`], @@ -104,6 +106,7 @@ export async function compileControlGraph(opts = {}) { cronIndexUrl, routePlanUrl, bundleUrl, + workflowDefinitionsUrl, }; } diff --git a/tests/helpers/load-control-shared.js b/tests/helpers/load-control-shared.js index 7ea75579..91a1e355 100644 --- a/tests/helpers/load-control-shared.js +++ b/tests/helpers/load-control-shared.js @@ -40,6 +40,8 @@ export function compileControlSharedDependencies({ sharedRedisUrl }) { "control/workflows-client.js", importSpecifierReplacements({ "shared-errors": SHARED_ERRORS_URL, + "shared-bounded-body": SHARED_BOUNDED_BODY_URL, + "shared-respond": SHARED_RESPOND_URL, "control-errors": controlErrorsUrl, }), ); diff --git a/tests/helpers/workflow-definitions.js b/tests/helpers/workflow-definitions.js new file mode 100644 index 00000000..de73abb5 --- /dev/null +++ b/tests/helpers/workflow-definitions.js @@ -0,0 +1,43 @@ +import { importSpecifierReplacements, repositoryFileUrl, repositoryModuleDataUrl } from "./load-shared-module.js"; +import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; + +export const workflowDefinitionsUrl = repositoryModuleDataUrl("control/workflow-definitions.js", importSpecifierReplacements({ + "shared-redis": sharedRedisStubUrl(), + "shared-utf8": repositoryFileUrl("shared/utf8.js"), +})); + +const definitions = await import(workflowDefinitionsUrl); + +/** @param {string} value */ +const bytes = (value) => Buffer.byteLength(value, "utf8"); + +/** @param {string} script @param {string[]} keys @param {unknown[]} args @param {import("./mocks/fake-redis.js").FakeRedisState} state */ +export async function workflowDefinitionRedisEval(script, keys, args, state) { + if (script === definitions.WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT && + state.hashes.get(keys[2])?.[String(args[3])] !== args[4]) { + return [2, null, [], 0]; + } + const hash = state.hashes.get(keys[0]) ?? {}; + const entries = Object.entries(hash); + if (script === definitions.WORKFLOW_ROUTE_PAGE_SCRIPT) { + const start = Number(args[0]); + const page = entries.length <= 128 ? entries : entries.slice(start, start + 32); + const next = entries.length <= 128 || start + page.length >= entries.length ? "0" : String(start + page.length); + const flat = page.flat(); + return [1, next, flat]; + } + const count = entries.length; + const size = entries.reduce((total, [field, value]) => total + bytes(field) + bytes(value), 0); + const fits = count <= Number(args[0]) && size <= Number(args[1]); + if (script === definitions.WORKFLOW_DEFINITION_ADMISSION_SCRIPT) { + return fits ? [1, count, size, args.slice(2).map((name) => Object.hasOwn(hash, String(name)) ? hash[String(name)] : null)] : [0, count, size, []]; + } + if (script === definitions.WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT) { + if (!fits) return [0, null, [], 0]; + const meta = state.hashes.get(keys[1])?.__meta__ ?? null; + const metaBytes = meta === null ? 0 : bytes(meta); + if (metaBytes > Number(args[2])) return [-1, null, [], 0]; + return [1, meta, entries.flat(), size + metaBytes]; + } + throw new Error("Unexpected Workflow Redis script"); +} diff --git a/tests/integration/auth-worker.test.js b/tests/integration/auth-worker.test.js index a67261d7..13fb22c8 100644 --- a/tests/integration/auth-worker.test.js +++ b/tests/integration/auth-worker.test.js @@ -55,7 +55,7 @@ test("whoami returns current principal without exposing token material", async ( assert.equal(ops.json.tokenId, "bootstrap"); assert.equal(typeof ops.json.requestId, "string"); assert.equal(ops.json.platformVersion, expectedPlatformVersion); - assert.equal(ops.json.minCliVersion, "1.8.0"); + assert.equal(ops.json.minCliVersion, "1.9.0"); assert.equal(Object.hasOwn(ops.json, "workerdVersion"), false); assert.deepEqual(ops.json.urls, { control: `http://${ADMIN_HOST_HEADER}`, @@ -73,7 +73,7 @@ test("whoami returns current principal without exposing token material", async ( assert.deepEqual(tenant.json.principal, { kind: "ns", ns }); assert.equal(tenant.json.tokenId, issued.json.tokenId); assert.equal(tenant.json.platformVersion, expectedPlatformVersion); - assert.equal(tenant.json.minCliVersion, "1.8.0"); + assert.equal(tenant.json.minCliVersion, "1.9.0"); assert.deepEqual(tenant.json.urls, { control: `http://${ADMIN_HOST_HEADER}`, namespace: `http://${ns}.workers.local`, diff --git a/tests/integration/cli-smoke.test.js b/tests/integration/cli-smoke.test.js index 1560b1b0..917d623e 100644 --- a/tests/integration/cli-smoke.test.js +++ b/tests/integration/cli-smoke.test.js @@ -5,20 +5,141 @@ import { mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import { sessionPolicyKey } from "../../shared/worker-contract.js"; import { withTempDir } from "../helpers/temp-dir.js"; -import { redisGetJson } from "./helpers/redis.js"; +import { workflowDefinitionsUrl } from "../helpers/workflow-definitions.js"; +import { redisDel, redisGetJson } from "./helpers/redis.js"; +import { WORKER_CODE, workerMeta, workflowInstanceStateKey } from "./helpers/workflows-scenarios.js"; import { adminGetFresh, assertOk, + deployAndPromote, gatewayFetch, parseStdoutJson, + readIntegrationJson, responseJson, runWdlCli, uniqueNs, setupIntegrationSuite, + waitUntil, } from "./helpers/index.js"; setupIntegrationSuite(); +const { WORKFLOW_DEFINITION_PAGE_MAX_WORKERS } = await import(workflowDefinitionsUrl); + +test("wdl CLI follows definition cursors through empty and short pages", async () => { + const ns = uniqueNs("wdl-wf-pages"); + const middleWorker = `page-${String(WORKFLOW_DEFINITION_PAGE_MAX_WORKERS).padStart(2, "0")}`; + for (let index = 0; index < 2 * WORKFLOW_DEFINITION_PAGE_MAX_WORKERS; index += 1) { + const name = `page-${String(index).padStart(2, "0")}`; + await deployAndPromote(ns, name, name === middleWorker ? { + code: WORKER_CODE, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + } : { code: 'export default { fetch() { return new Response("ok"); } };' }); + } + await deployAndPromote(ns, "z-flow", { code: WORKER_CODE, workflows: [ + { name: "orders", binding: "ORDERS", className: "OrderWorkflow" }, + { name: "refunds", binding: "REFUNDS", className: "OrderWorkflow" }, + ] }); + + const expected = [[], [`${middleWorker}/orders`], ["z-flow/orders", "z-flow/refunds"]]; + /** @type {string | null} */ + let cursor = null; + let pages = 0; + do { + assert.ok(pages < expected.length, "definition pagination must terminate"); + const args = ["workflows", "list", "--ns", ns, "--limit", "2"]; + if (cursor !== null) args.push("--cursor", cursor); + const json = runWdlCli([...args, "--json"]); + assertOk(json); + const page = parseStdoutJson(json.stdout, "Workflow definition page"); + assert.equal(page.namespace, ns); + assert.deepEqual(page.workflows.map((/** @type {{worker: string, name: string}} */ entry) => + `${entry.worker}/${entry.name}`), expected[pages]); + const human = runWdlCli(args); + assertOk(human); + if (pages === 0) assert.match(human.stdout, /^\(no workflows on this page\)$/m); + for (const name of expected[pages]) assert.ok(human.stdout.includes(`${name}\t`)); + if (pages < expected.length - 1) { + assert.equal(typeof page.cursor, "string"); + assert.ok(page.cursor.length > 0); + assert.notEqual(page.cursor, cursor); + assert.ok(human.stdout.split(/\r?\n/).includes(`Next cursor: ${page.cursor}`)); + } else { + assert.equal(page.cursor, null); + assert.doesNotMatch(human.stdout, /^Next cursor:/m); + } + cursor = page.cursor; + pages += 1; + } while (cursor !== null); + assert.equal(pages, expected.length); +}); + +test("wdl CLI follows instance cursors after an empty page without skipping results", async () => { + const ns = uniqueNs("wdl-wf-instances"); + const version = await deployAndPromote(ns, "shop", { + code: WORKER_CODE, + vars: { LABEL: "cli-pagination" }, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + const ids = ["item-0", "item-1", "item-2"]; + for (const id of ids) { + const created = await readIntegrationJson(await gatewayFetch(ns, `/shop/create?id=${id}`), 200); + assert.equal(created.id, id); + } + const command = ["workflows", "instances", "shop", "orders", "--ns", ns]; + await waitUntil("CLI pagination fixtures have durable outputs", async () => { + const result = runWdlCli([...command, "--json"]); + assertOk(result); + const page = parseStdoutJson(result.stdout, "Workflow instances before pagination"); + return page.instances.length === ids.length && page.instances.every( + (/** @type {{status: string}} */ instance) => instance.status === "completed" + ); + }); + // Model a state disappearing after its rank-index entry was selected. + const workflowKey = workerMeta(ns, "shop", version).workflows[0].workflowKey; + redisDel(workflowInstanceStateKey(ns, workflowKey, ids[0]), { db: 2 }); + + /** @type {string[]} */ + const seen = []; + /** @type {string | null} */ + let cursor = null; + let pages = 0; + do { + assert.ok(pages < ids.length, "instance pagination must terminate"); + const args = [...command, "--limit", "1"]; + if (cursor !== null) args.push("--cursor", cursor); + const json = runWdlCli([...args, "--json"]); + assertOk(json); + const page = parseStdoutJson(json.stdout, "Workflow instance page"); + const human = runWdlCli(args); + assertOk(human); + if (pages === 0) { + assert.deepEqual(page.instances, []); + assert.match(human.stdout, /^\(no workflow instances on this page\)$/m); + } else { + assert.equal(page.instances.length, 1); + const instance = page.instances[0]; + assert.equal(instance.id, ids[pages]); + assert.equal(instance.status, "completed"); + assert.equal(instance.output.instanceId, instance.id); + assert.equal(instance.output.fromEnv, "cli-pagination"); + assert.ok(human.stdout.split(/\r?\n/).includes(`${instance.id}\tstatus=completed`)); + seen.push(instance.id); + } + if (pages < ids.length - 1) { + assert.equal(page.cursor, String(pages + 1)); + assert.ok(human.stdout.split(/\r?\n/).includes(`Next cursor: ${page.cursor}`)); + } else { + assert.equal(page.cursor, null); + assert.doesNotMatch(human.stdout, /^Next cursor:/m); + } + cursor = page.cursor; + pages += 1; + } while (cursor !== null); + assert.equal(pages, ids.length); + assert.deepEqual(seen, ids.slice(1)); +}); + test("wdl CLI exercises deploy, workers, secrets, and delete lifecycle", async () => { const ns = uniqueNs("wdl-smoke"); diff --git a/tests/integration/helpers/workflows-scenarios.js b/tests/integration/helpers/workflows-scenarios.js index e6886843..22605118 100644 --- a/tests/integration/helpers/workflows-scenarios.js +++ b/tests/integration/helpers/workflows-scenarios.js @@ -14,6 +14,7 @@ import { readMeta } from "./misc.js"; import { redisCommandCalls, redisDel, + redisEval, redisHDel, redisHGet, redisHSet, @@ -64,6 +65,42 @@ export const WORKER_CODE = readFileSync( "utf8" ); +/** @param {string} ns @param {string} worker */ +export function workflowByWorkerKey(ns, worker) { + return `wf:by-worker:${ns}:${worker}`; +} + +/** @param {string} ns @param {string} worker @param {string} version */ +export function workflowByVersionKey(ns, worker, version) { + return `wf:by-version:${ns}:${worker}:${version}`; +} + +/** @param {string} ns @param {string} worker @param {string} version */ +export function workflowPendingVersionKey(ns, worker, version) { + return `wf:pending-version:${ns}:${worker}:${version}`; +} + + +/** @param {string} ns @param {string} workflowKey @param {string} version @param {number} pending @param {number} stale */ +export function seedWorkflowLifecycleMembers(ns, workflowKey, version, pending, stale) { + return redisEval(` +for i = 1, tonumber(ARGV[4]) do + local id = 'pending-' .. i + local key = 'wf:instance:{' .. ARGV[1] .. ':' .. ARGV[2] .. ':' .. id .. '}:state' + redis.call('HSET', key, 'ns', ARGV[1], 'worker', 'shop', 'workflowKey', ARGV[2], + 'instanceId', id, 'frozenVersion', ARGV[3], 'workflowName', 'orders', + 'className', 'OrderWorkflow', 'status', 'pending_create', 'generation', '1', + 'createdAtMs', '1', 'pendingCreateToken', id, 'pendingExpiresAtMs', '1') + redis.call('SADD', KEYS[1], ARGV[2] .. '\\t' .. id) + redis.call('SADD', KEYS[2], ARGV[2] .. '\\t' .. id) +end +for i = 1, tonumber(ARGV[5]) do + redis.call('SADD', KEYS[1], ARGV[2] .. '\\tstale-' .. i) +end +return 1`, [workflowByWorkerKey(ns, "shop"), workflowByVersionKey(ns, "shop", version)], + [ns, workflowKey, version, String(pending), String(stale)], { db: 2 }); +} + /** @param {string} ns @param {string} worker @param {string} version */ export function workerMeta(ns, worker, version) { return readMeta(ns, worker, version); diff --git a/tests/integration/workflows-metadata.test.js b/tests/integration/workflows-metadata.test.js index 87bb2a43..c64e3005 100644 --- a/tests/integration/workflows-metadata.test.js +++ b/tests/integration/workflows-metadata.test.js @@ -1,15 +1,21 @@ // WDL Workflows metadata path: deploy-time parsing and Redis persistence. -// Execution is covered by later workflows milestones. +// Execution is covered by the Workflow runtime suites. import { test } from "node:test"; import assert from "node:assert/strict"; import { adminPost, + adminFetch, + readIntegrationJson, assertStatus, readMeta, uniqueNs, setupIntegrationSuite, } from "./helpers/index.js"; -import { redisHGetJson } from "./helpers/redis.js"; +import { redisHGetJson, redisEval, redisHSet, redisHGetAll, redisHStrLen, redisJsonMember } from "./helpers/redis.js"; +import { bundleKey, routesKey } from "../../shared/worker-contract.js"; +import { workflowDefinitionsUrl } from "../helpers/workflow-definitions.js"; + +const definitions = await import(workflowDefinitionsUrl); setupIntegrationSuite(); @@ -89,3 +95,97 @@ test("deploy rejects Cloudflare script_name workflows with a stable code", async assert.equal(res.status, 400); assert.equal(res.json.error, "workflow_script_name_unsupported"); }); + +test("definition snapshot atomically checks the route and returns stored byte totals", async () => { + const ns = uniqueNs("wfmetasnapshot"); + const deployed = await adminPost(`/ns/${ns}/worker/shop/deploy`, { + code: WORKER_CODE, + vars: { LABEL: "\u4e2d\u6587" }, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + assertStatus(deployed, 201); + const version = deployed.json.version; + assertStatus(await adminPost(`/ns/${ns}/worker/shop/promote`, { version }), 200); + const defsKey = `wf:defs:${ns}:shop`; + const metaKey = bundleKey(ns, "shop", version); + const expectedBytes = redisHStrLen(metaKey, "__meta__") + Object.entries(redisHGetAll(defsKey)) + .reduce((total, [field, value]) => total + Buffer.byteLength(field) + Buffer.byteLength(value), 0); + const script = `local snapshot = (function()\n${definitions.WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT}\nend)() +return cjson.encode({status=snapshot[1], bytes=snapshot[4], hasMeta=type(snapshot[2])=='string', hasDefs=next(snapshot[3])~=nil})`; + const args = [String(definitions.WORKFLOW_DEFINITIONS_MAX_COUNT), String(definitions.WORKFLOW_DEFINITIONS_MAX_BYTES), String(definitions.WORKFLOW_LIST_META_MAX_BYTES), "shop", version]; + const keys = [defsKey, metaKey, routesKey(ns)]; + const snapshot = redisJsonMember(redisEval(script, keys, args), "Workflow snapshot"); + assert.deepEqual(snapshot, { status: 1, bytes: expectedBytes, hasMeta: true, hasDefs: true }); + args[4] = "v999"; + const changed = redisJsonMember(redisEval(script, keys, args), "changed Workflow snapshot"); + assert.deepEqual(changed, { status: 2, bytes: 0, hasMeta: false, hasDefs: false }); +}); + +test("Workflow definition listing pages retained names and deploy admission bounds their growth", async () => { + const ns = uniqueNs("wfdefpage"); + const first = await adminPost(`/ns/${ns}/worker/shop/deploy`, { code: WORKER_CODE, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + assertStatus(first, 201); + const promoted = await adminPost(`/ns/${ns}/worker/shop/promote`, { version: first.json.version }); + assertStatus(promoted, 200); + const key = `wf:defs:${ns}:shop`; + redisEval(` +for i = 1, 1023 do + redis.call('HSET', KEYS[1], 'retired-' .. i, + cjson.encode({workflowKey='wf_' .. string.format('%032x', i), className='OrderWorkflow'})) +end +return 1`, [key], []); + const names = new Set(); + let cursor = ""; + let pages = 0; + do { + const path = `/ns/${ns}/workflows?limit=200${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`; + const body = await readIntegrationJson(await adminFetch(path), 200); + assert.ok(body.workflows.length <= 200); + for (const entry of body.workflows) { + assert.equal(names.has(entry.name), false); + names.add(entry.name); + assert.equal(entry.retired === true, entry.name !== "orders"); + } + cursor = body.cursor; + pages += 1; + assert.ok(pages < 10); + } while (cursor !== null); + assert.equal(names.size, 1024); + assert.equal(pages, 6); + const rejected = await adminPost(`/ns/${ns}/worker/shop/deploy`, { code: WORKER_CODE, + workflows: [{ name: "extra", binding: "EXTRA", className: "OrderWorkflow" }], + }); + assertStatus(rejected, 413); + assert.equal(rejected.json.error, "workflow_definitions_too_large"); + assert.equal(Number(redisEval("return redis.call('HLEN', KEYS[1])", [key], [])), 1024); + const replaced = await adminPost(`/ns/${ns}/worker/shop/deploy`, { code: WORKER_CODE, + workflows: [{ name: "orders", binding: "ORDERS", className: "ReplacementWorkflow" }], + }); + assertStatus(replaced, 201); + assert.equal(Number(redisEval("return redis.call('HLEN', KEYS[1])", [key], [])), 1024); +}); + +test("Workflow definition readers bound stored bytes before returning them from Valkey", async () => { + const ns = uniqueNs("wfdefbytes"); + const first = await adminPost(`/ns/${ns}/worker/shop/deploy`, { code: WORKER_CODE, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + assertStatus(first, 201); + assertStatus(await adminPost(`/ns/${ns}/worker/shop/promote`, { version: first.json.version }), 200); + const defs = `wf:defs:${ns}:shop`; + redisEval("redis.call('HSET', KEYS[1], 'oversized', string.rep('x', 1048577)); return 1", [defs], []); + const invalid = await readIntegrationJson(await adminFetch(`/ns/${ns}/workflows`), 500); + assert.equal(invalid.error, "corrupt_meta"); + redisEval("redis.call('HDEL', KEYS[1], 'oversized'); return 1", [defs], []); + const bundle = bundleKey(ns, "shop", first.json.version); + const meta = readMeta(ns, "shop", first.json.version); + try { + redisEval("redis.call('HSET', KEYS[1], '__meta__', string.rep('x', 8388609)); return 1", [bundle], []); + const tooLarge = await readIntegrationJson(await adminFetch(`/ns/${ns}/workflows`), 413); + assert.equal(tooLarge.error, "workflow_listing_metadata_too_large"); + } finally { + redisHSet(bundle, { __meta__: JSON.stringify(meta) }); + } +}); diff --git a/tests/integration/workflows-runtime-core.test.js b/tests/integration/workflows-runtime-core.test.js index 1b16dbeb..17004442 100644 --- a/tests/integration/workflows-runtime-core.test.js +++ b/tests/integration/workflows-runtime-core.test.js @@ -6,15 +6,21 @@ import { KV_READ_INFRASTRUCTURE_ERROR_CODE, WORKFLOW_INFRASTRUCTURE_REPORT_ORIGIN, } from "../../runtime/infrastructure-error.js"; -import { readRepositoryModuleSource } from "../helpers/load-shared-module.js"; -import { prometheusCounter } from "./helpers/prometheus.js"; +import { importSpecifierReplacements, readRepositoryJson, readRepositoryModuleSource } from "../helpers/load-shared-module.js"; +import { redisEval } from "./helpers/redis.js"; +import { sh } from "./helpers/cli.js"; +import { parseCounters, prometheusCounter } from "./helpers/prometheus.js"; import { WORKER_CODE, + adminFetch, + delay, deployAndPromote, dispatchWorkflowReplay, gatewayFetch, gatewayWorkerId, redisSMembers, + redisDel, + redisWorkflowStateHGet, redisWorkflowStateHDel, redisWorkflowStateHSet, redisZAdd, @@ -30,11 +36,62 @@ import { uniqueNs, waitUntil, workflowEventTypeIndexKey, + workflowInstanceStateKey, workerMeta, } from "./helpers/workflows-scenarios.js"; setupIntegrationSuite(); +const REPLAY_CAPACITY_WORKER = readRepositoryModuleSource("test-workers/workflow-replay-capacity/src/index.js"); + +test("closing a Workflow controller releases a pending replay lease across real JSRPC", async () => { + const ns = uniqueNs("wfreplayclose"); + await deployAndPromote(ns, "root", { + code: readRepositoryModuleSource("test-workers/workflow-replay-lifetime/src/root.js"), + }); + assert.deepEqual(await readIntegrationJson(await gatewayFetch(ns, "/root/"), 200), { ready: true }); + const sources = { + "workflow-step.js": "runtime/dispatch/workflow-step.js", + "workflow-replay-cache.js": "runtime/dispatch/workflow-replay-cache.js", + "workflow-json.js": "runtime/dispatch/workflow-json.js", + "bounded-body.js": "shared/bounded-body.js", + "respond.js": "shared/respond.js", + "internal-auth.js": "shared/internal-auth.js", + "utf8.js": "shared/utf8.js", + "metrics.js": "test-workers/workflow-replay-lifetime/src/metrics.js", + "worker.js": "test-workers/workflow-replay-lifetime/src/index.js", + }; + const replacements = importSpecifierReplacements({ + "runtime-dispatch-workflow-json": "./workflow-json.js", + "runtime-dispatch-workflow-replay-cache": "./workflow-replay-cache.js", + "runtime-metrics": "./metrics.js", + "shared-bounded-body": "./bounded-body.js", + "shared-respond": "./respond.js", + "shared-internal-auth": "./internal-auth.js", + "shared-utf8": "./utf8.js", + }); + await deployAndPromote(ns, "probe", { + mainModule: "worker.js", + modules: Object.fromEntries(Object.entries(sources).map(([name, source]) => [name, readRepositoryModuleSource(source, replacements)])), + bindings: { ROOT: { type: "service", service: "root" } }, + }); + for (let iteration = 1; iteration <= 3; iteration += 1) { + const response = await readIntegrationJson(await gatewayFetch(ns, "/probe/run"), 503); + assert.match(response.rootError, /^root escaped\b/); + assert.equal(response.before.bytes, 1024 * 1024); + assert.equal(response.after.bytes, 0); + assert.equal(response.after.invocations, iteration); + assert.equal(response.after.cancellations, iteration); + } + await delay(2_100); + const after = await readIntegrationJson(await gatewayFetch(ns, "/probe/state"), 200); + assert.deepEqual(after, { invocations: 3, cancellations: 3, bytes: 0 }); +}); + +const workflowLimits = /** @type {{ instancesResponseBytesMax: number, resultBytesMax: number }} */ ( + readRepositoryJson("tests/fixtures/workflow-limits.json") +); + const WORKFLOW_KV_PROVENANCE_PROBE_CODE = readRepositoryModuleSource( "test-workers/workflow-kv-provenance/src/index.js", [ @@ -50,6 +107,167 @@ const WORKFLOW_KV_PROVENANCE_PROBE_CODE = readRepositoryModuleSource( ] ); +test("concurrent large Workflow replay stays admitted and releases detached state", async (t) => { + const ns = uniqueNs("wfreplaycap"); + const version = await deployAndPromote(ns, "shop", { + code: REPLAY_CAPACITY_WORKER, + bindings: { STATE: { type: "kv", id: "7".repeat(32) } }, + workflows: [{ name: "capacity", binding: "CAPACITY", className: "ReplayCapacityWorkflow" }], + }); + const workflowKey = workerMeta(ns, "shop", version).workflows[0].workflowKey; + const metrics = () => serviceInternalGet("user-runtime", 8088, "/_metrics").body; + const gauge = (/** @type {string} */ body, /** @type {string} */ name) => { + const value = parseCounters(body).get(`wdl_${name}`); + assert.ok(typeof value === "number", `missing replay gauge ${name}`); + return value; + }; + const saturated = (/** @type {string} */ body) => prometheusCounter(body, "wdl_workflow_replay_cache_total", { outcome: "saturated" }); + const before = saturated(metrics()); + // RSS sampling is optional; replay budget and release assertions are not. + const memory = (read = sh) => { + try { + const container = read(["docker", "compose", "ps", "-q", "user-runtime"]).trim(); + if (!container) return "unavailable"; + return read(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container]).trim(); + } catch { + return "unavailable"; + } + }; + for (const failedRead of [1, 2]) { + let reads = 0; + assert.equal(memory(() => { + if (++reads === failedRead) throw new Error("optional Docker diagnostic failed"); + return "test-container"; + }), "unavailable"); + } + t.diagnostic(`Runtime container memory before replay pressure: ${memory()}`); + const entries = Array.from({ length: 20 }, (_, index) => ({ instanceId: `capacity-${index}`, params: { id: `capacity-${index}` } })); + const created = serviceInternalPost("workflows", 9120, "/internal/workflows/create-batch", { + ns, worker: "shop", frozenVersion: version, workflowName: "capacity", workflowKey, + className: "ReplayCapacityWorkflow", entries, + }); + assert.equal(created.status, 200, created.body); + try { + await waitUntil("replay pressure is visible while roots remain active", async () => { + const body = metrics(); + assert.ok(gauge(body, "workflow_replay_working_set_bytes") <= 64 * 1024 * 1024); + return saturated(body) > before && gauge(body, "workflow_replay_detached_bytes") > 0; + }); + const body = metrics(); + assert.ok(gauge(body, "workflow_replay_working_set_high_water_bytes") > 32 * 1024 * 1024); + assert.ok(gauge(body, "workflow_replay_working_set_high_water_bytes") <= 64 * 1024 * 1024); + t.diagnostic(`Runtime container memory with admitted replay pressure: ${memory()}`); + } finally { + await readIntegrationJson(await gatewayFetch(ns, "/shop/release"), 200, "release replay roots"); + } + await waitUntil("all replay-capacity instances recover and finish", async () => Number(redisEval(` +local completed = 0 +for _, key in ipairs(KEYS) do + if redis.call('HGET', key, 'status') == 'completed' then completed = completed + 1 end +end +return completed`, entries.map((entry) => workflowInstanceStateKey(ns, workflowKey, entry.instanceId)), [], { db: 2 })) === entries.length); + for (const entry of entries) { + const response = serviceInternalPost("workflows", 9120, "/internal/workflows/status", { + ns, worker: "shop", frozenVersion: version, workflowName: "capacity", workflowKey, + className: "ReplayCapacityWorkflow", instanceId: entry.instanceId, + }); + assert.equal(response.status, 200, response.body); + assert.deepEqual(responseJson(response).output, { id: entry.instanceId, callbacks: 6 }); + } + await waitUntil("replay readers and detached caches are released", async () => { + const body = metrics(); + return gauge(body, "workflow_replay_active_bytes") === 0 && gauge(body, "workflow_replay_detached_bytes") === 0 && gauge(body, "workflow_replay_read_in_flight_bytes") === 0; + }); + assert.ok(gauge(metrics(), "workflow_replay_working_set_bytes") <= 16 * 1024 * 1024); + t.diagnostic(`Runtime container memory after replay roots completed: ${memory()}`); +}); + +test("workflow instance pages bound payload bytes without skipping an index gap or duplicate ref", async () => { + const ns = uniqueNs("wflist"); + const version = await deployAndPromote(ns, "shop", { + code: ` +import { WorkflowEntrypoint } from "cloudflare:workers"; +export class OrderWorkflow extends WorkflowEntrypoint { + async run(event) { return "x".repeat(event.payload.bytes); } +} +export default { fetch() { return new Response("ok"); } }; +`, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + const workflowKey = workerMeta(ns, "shop", version).workflows[0].workflowKey; + const entries = Array.from({ length: 40 }, (_, index) => ({ + instanceId: `item-${String(index).padStart(2, "0")}`, + params: { bytes: index < 12 ? workflowLimits.resultBytesMax - 2 : 16 }, + })); + const created = serviceInternalPost("workflows", 9120, "/internal/workflows/create-batch", { + ns, worker: "shop", frozenVersion: version, workflowName: "orders", + workflowKey, className: "OrderWorkflow", entries, + }); + assert.equal(created.status, 200, created.body); + await waitUntil("all list payload fixtures are durable", async () => { + const completed = redisEval(` +local completed = 0 +for _, key in ipairs(KEYS) do + if redis.call('HGET', key, 'status') == 'completed' then completed = completed + 1 end +end +return completed`, entries.map((entry) => workflowInstanceStateKey(ns, workflowKey, entry.instanceId)), [], { db: 2 }); + return Number(completed) === entries.length; + }); + + // Keep an index gap and a duplicate payload reference across a 32-ref read boundary. + redisDel(workflowInstanceStateKey(ns, workflowKey, entries[1].instanceId), { db: 2 }); + const last = entries.at(-1); + assert.ok(last); + redisWorkflowStateHSet(ns, workflowKey, last.instanceId, [ + "errorRef", redisWorkflowStateHGet(ns, workflowKey, last.instanceId, "outputRef"), + ]); + + const expected = entries.filter((_, index) => index !== 1); + /** @type {string[]} */ + const seen = []; + /** @type {string | null} */ + let cursor = ""; + let pages = 0; + do { + const response = await adminFetch(`/ns/${ns}/workflows/shop/orders/instances?limit=1000&cursor=${cursor}`); + const text = await response.text(); + assert.equal(response.status, 200, text.slice(0, 500)); + assert.ok(Buffer.byteLength(text) <= workflowLimits.instancesResponseBytesMax); + const page = responseJson({ body: text }); + assert.ok(page.instances.length > 0); + for (const instance of page.instances) { + const entry = expected[seen.length]; + assert.equal(instance.id, entry.instanceId); + assert.equal(instance.status, "completed"); + assert.equal(instance.output, "x".repeat(entry.params.bytes)); + assert.equal(instance.error, entry === last ? instance.output : null); + seen.push(instance.id); + } + if (pages === 0) assert.equal(page.cursor, "8"); + assert.ok(page.cursor === null || Number(page.cursor) > Number(cursor)); + cursor = page.cursor; + pages += 1; + assert.ok(pages <= entries.length); + } while (cursor !== null); + assert.equal(pages, 2); + assert.deepEqual(seen, expected.map((entry) => entry.instanceId)); + + const payloadsKey = redisWorkflowStateHGet(ns, workflowKey, last.instanceId, "payloadsKey"); + const outputRef = redisWorkflowStateHGet(ns, workflowKey, last.instanceId, "outputRef"); + redisEval(` +redis.call('HSET', KEYS[1], ARGV[1], '"' .. string.rep('x', tonumber(ARGV[2]) - 1) .. '"') +return 1`, [payloadsKey], [outputRef, String(workflowLimits.resultBytesMax)], { db: 2 }); + for (const endpoint of ["get", "status", "instances"]) { + const invalid = serviceInternalPost("workflows", 9120, `/internal/workflows/${endpoint}`, { + ns, worker: "shop", frozenVersion: version, workflowName: "orders", + workflowKey, className: "OrderWorkflow", instanceId: last.instanceId, + options: { cursor: "39" }, + }); + assert.equal(invalid.status, 500, invalid.body); + assert.equal(responseJson(invalid).error, "workflow_invalid_state"); + } +}); + test("workflow binding creates and reads an instance through workflows", async () => { const ns = uniqueNs("wfrt"); const version = await deployAndPromote(ns, "shop", { diff --git a/tests/integration/workflows-runtime-retention.test.js b/tests/integration/workflows-runtime-retention.test.js index b0bc2c19..180098df 100644 --- a/tests/integration/workflows-runtime-retention.test.js +++ b/tests/integration/workflows-runtime-retention.test.js @@ -1,6 +1,7 @@ // WDL Workflows retention, frozen-version, and delete-blocker paths. import { test } from "node:test"; import assert from "node:assert/strict"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { WORKER_CODE, adminFetch, @@ -24,12 +25,171 @@ import { waitUntil, withServiceStopped, workflowRetentionKey, + workflowByWorkerKey, + workflowInstanceStateKey, + workflowPendingVersionKey, + seedWorkflowLifecycleMembers, workerMeta, } from "./helpers/workflows-scenarios.js"; -import { redisZAdd, redisZScore } from "./helpers/redis.js"; +import { redisZAdd, redisZScore, redisSCard, redisCommandCalls, redisEval, redisHGetAll, redisHSet, redisZCard } from "./helpers/redis.js"; setupIntegrationSuite(); +test("Workflow restart rejects invalid persisted params before any mutation", async () => { + const ns = uniqueNs("wfrestartparams"); + const version = await deployAndPromote(ns, "shop", { code: WORKER_CODE, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + const workflowKey = workerMeta(ns, "shop", version).workflows[0].workflowKey; + const instanceId = "params"; + await readIntegrationJson(await gatewayFetch(ns, `/shop/create?id=${instanceId}`), 200); + await waitUntil("restart params fixture completes", async () => + redisWorkflowStateHGet(ns, workflowKey, instanceId, "status") === "completed"); + const stateKey = workflowInstanceStateKey(ns, workflowKey, instanceId); + const payloadsKey = redisWorkflowStateHGet(ns, workflowKey, instanceId, "payloadsKey"); + const paramsRef = redisWorkflowStateHGet(ns, workflowKey, instanceId, "paramsRef"); + const markerKey = workflowPendingVersionKey(ns, "shop", version); + const { paramsBytesMax } = /** @type {{paramsBytesMax: number}} */ (readRepositoryJson("tests/fixtures/workflow-limits.json")); + const request = { ns, worker: "shop", frozenVersion: version, workflowName: "orders", workflowKey, className: "OrderWorkflow", instanceId }; + await withServiceStopped("scheduler", async () => { + const before = redisHGetAll(stateKey, { db: 2 }); + for (const invalid of ["oversized", "malformed"]) { + if (invalid === "oversized") { + redisEval(`return redis.call('HSET', KEYS[1], ARGV[1], '"' .. string.rep('x', tonumber(ARGV[2]) - 1) .. '"')`, + [payloadsKey], [paramsRef, String(paramsBytesMax)], { db: 2 }); + } else { + redisHSet(payloadsKey, { [paramsRef]: "{broken" }, { db: 2 }); + } + const writes = redisCommandCalls("zadd"); + const result = serviceInternalPost("workflows", 9120, "/internal/workflows/restart", request); + assert.equal(result.status, 500, result.body); + assert.equal(responseJson(result).error, "workflow_invalid_state"); + assert.deepEqual(redisHGetAll(stateKey, { db: 2 }), before); + assert.equal(redisZCard(markerKey, { db: 2 }), 0); + assert.equal(redisCommandCalls("zadd"), writes, "invalid params must fail before creating a pending-restart marker"); + } + const digest = redisEval(` +local prefix = '{ "id":"params", "number":1e+0, "padding":"' +local raw = prefix .. string.rep('x', tonumber(ARGV[2]) - #prefix - 2) .. '"}' +redis.call('HSET', KEYS[1], ARGV[1], raw) +return redis.sha1hex(raw)`, [payloadsKey], [paramsRef, String(paramsBytesMax)], { db: 2 }); + const restarted = serviceInternalPost("workflows", 9120, "/internal/workflows/restart", request); + assert.equal(restarted.status, 200, restarted.body); + assert.equal(responseJson(restarted).status, "queued"); + assert.equal(redisWorkflowStateHGet(ns, workflowKey, instanceId, "generation"), "2"); + assert.equal(redisEval("return redis.sha1hex(redis.call('HGET', KEYS[1], 'params'))", [payloadsKey], [], { db: 2 }), digest); + }); + await waitUntil("exact-limit params dispatch after restart", async () => + redisWorkflowStateHGet(ns, workflowKey, instanceId, "status") === "completed"); +}); + +test("Workflow lifecycle dry-run stays bounded and cleanup resumes across pages", async () => { + const ns = uniqueNs("wflifepage"); + const version = await deployAndPromote(ns, "shop", { code: WORKER_CODE, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + const workflowKey = workerMeta(ns, "shop", version).workflows[0].workflowKey; + seedWorkflowLifecycleMembers(ns, workflowKey, version, 2000, 0); + const beforeReads = redisCommandCalls("hgetall"); + const dry = serviceInternalPost("workflows", 9120, "/internal/workflows/lifecycle/check-delete", { ns, worker: "shop" }); + assert.equal(dry.status, 200, dry.body); + const dryBody = responseJson(dry); + assert.equal(dryBody.allowed, false); + assert.equal(dryBody.blockers.length, 20); + assert.ok(redisCommandCalls("hgetall") - beforeReads < 1000, "dry-run must not materialize the complete pending set"); + assert.equal(redisSCard(workflowByWorkerKey(ns, "shop"), { db: 2 }), 2000); + seedWorkflowLifecycleMembers(ns, workflowKey, version, 0, 2000); + const partial = serviceInternalPost("workflows", 9120, "/internal/workflows/lifecycle/check-delete", { ns, worker: "shop", allowCleanup: true }); + assert.equal(partial.status, 200, partial.body); + const partialBody = responseJson(partial); + assert.equal(partialBody.allowed, false); + assert.equal(partialBody.blockers.length, 0); + assert.equal(typeof partialBody.cursor, "string"); + const remaining = redisSCard(workflowByWorkerKey(ns, "shop"), { db: 2 }); + assert.ok(remaining > 0 && remaining < 4000); + await waitUntil("bounded cleanup eventually permits deletion", async () => { + const response = await adminFetch(`/ns/${ns}/worker/shop/delete`, { method: "POST" }); + if (response.status === 503) { + assert.equal((await readIntegrationJson(response, 503)).error, "workflow_lifecycle_check_incomplete"); + return false; + } + await readIntegrationJson(response, 200); + return true; + }); + assert.equal(redisSCard(workflowByWorkerKey(ns, "shop"), { db: 2 }), 0); +}); + +test("Workflow lifecycle continuation never skips a live blocker among stale members", async () => { + const ns = uniqueNs("wflivepage"); + const version = await deployAndPromote(ns, "shop", { code: WORKER_CODE, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + const workflowKey = workerMeta(ns, "shop", version).workflows[0].workflowKey; + await readIntegrationJson(await gatewayFetch(ns, "/shop/create?id=live&wait=1&noWaitTimeout=1&retentionMs=1000"), 200); + await waitUntil("live deletion blocker waits", async () => redisWorkflowStateHGet(ns, workflowKey, "live", "status") === "waiting"); + seedWorkflowLifecycleMembers(ns, workflowKey, version, 0, 2000); + let blocked = false; + for (let attempt = 0; attempt < 4 && !blocked; attempt += 1) { + const response = await adminFetch(`/ns/${ns}/worker/shop/delete`, { method: "POST" }); + if (response.status === 503) { + assert.equal((await readIntegrationJson(response, 503)).error, "workflow_lifecycle_check_incomplete"); + continue; + } + const body = await readIntegrationJson(response, 409); + assert.equal(body.error, "workflow_instances_active"); + assert.ok(body.blockers.some((/** @type {{instanceId: string}} */ entry) => entry.instanceId === "live")); + blocked = true; + } + assert.equal(blocked, true); + assert.equal(redisWorkflowStateHGet(ns, workflowKey, "live", "status"), "waiting"); + await readIntegrationJson(await gatewayFetch(ns, "/shop/terminate?id=live"), 200); + await waitUntil("retention removes the final live blocker", async () => { + const response = await adminFetch(`/ns/${ns}/worker/shop/delete`, { method: "POST" }); + if (response.status === 409 || response.status === 503) return false; + await readIntegrationJson(response, 200); + return true; + }); +}); + +test("Workflow lifecycle final membership check requires a fresh scan after skipped pages", async () => { + const ns = uniqueNs("wfliferescan"); + const version = await deployAndPromote(ns, "shop", { code: WORKER_CODE, + workflows: [{ name: "orders", binding: "ORDERS", className: "OrderWorkflow" }], + }); + const workflowKey = workerMeta(ns, "shop", version).workflows[0].workflowKey; + seedWorkflowLifecycleMembers(ns, workflowKey, version, 0, 2000); + const key = workflowByWorkerKey(ns, "shop"); + const contract = /** @type {{limits: {scanCount: number}, responses: {rescanRequired: Record}}} */ ( + readRepositoryJson("tests/fixtures/workflow-lifecycle-check.json") + ); + // Resume at the last native scan page while retaining members from earlier pages. + const cursor = redisEval(` +local cursor = '0' +repeat + local page = redis.call('SSCAN', KEYS[1], cursor, 'COUNT', ARGV[1]) + if page[1] == '0' then return cursor end + cursor = page[1] +until false`, [key], [String(contract.limits.scanCount)], { db: 2 }); + assert.notEqual(cursor, "0"); + const response = serviceInternalPost("workflows", 9120, "/internal/workflows/lifecycle/check-delete", { + ns, worker: "shop", allowCleanup: true, cursor, + }); + assert.equal(response.status, 200, response.body); + const remaining = redisSCard(key, { db: 2 }); + assert.ok(remaining > 0); + assert.deepEqual(responseJson(response), { ...contract.responses.rescanRequired, count: remaining }); + await waitUntil("fresh cleanup traverses the remaining referrers", async () => { + const deleted = await adminFetch(`/ns/${ns}/worker/shop/delete`, { method: "POST" }); + if (deleted.status === 503) { + assert.equal((await readIntegrationJson(deleted, 503)).error, "workflow_lifecycle_check_incomplete"); + return false; + } + await readIntegrationJson(deleted, 200); + return true; + }); + assert.equal(redisSCard(key, { db: 2 }), 0); +}); + test("stale workflow run cannot commit after restart generation changes", async () => { const ns = uniqueNs("wfstale"); const version = await deployAndPromote(ns, "shop", { diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index e8194e9e..904ff3a2 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -697,7 +697,7 @@ test("worker delete reports cleanup_queue_failed when data-plane cleanup enqueue ); assert.deepEqual(testState.lockKinds, ["whole"]); assert.deepEqual(testState.workflowChecks, [{ - ns: "demo", worker: "api", allowCleanup: true, requestId: "rid-delete", + ns: "demo", worker: "api", allowCleanup: true, requestId: "rid-delete", lockToken: "lock-token", }]); assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old", requestId: "rid-delete", @@ -1477,6 +1477,7 @@ test("worker delete reports workflow and version referrer blockers together", as worker: "api", allowCleanup: true, requestId: "rid-delete-both-blockers", + lockToken: "lock-token", }]); assert.deepEqual(testState.doAlarmCleanups, []); assert.equal(testState.releaseCalls, 1); @@ -1517,6 +1518,7 @@ test("worker delete keeps DO alarm jobs when version referrers block deletion", worker: "api", allowCleanup: true, requestId: "rid-delete-version-blocker", + lockToken: "lock-token", }]); assert.deepEqual(testState.doAlarmCleanups, []); assert.equal(testState.multiCalls.length, 0); @@ -1591,6 +1593,7 @@ test("worker delete keeps workflow blocker even when worker lifecycle is already worker: "api", allowCleanup: true, requestId: "rid-delete-workflow-no-lifecycle", + lockToken: "lock-token", }]); assert.equal(testState.releaseCalls, 1); assert.deepEqual(testState.cleanupIntents, []); @@ -1786,7 +1789,7 @@ test("worker delete reports queueHint none when no content cleanup is needed", a const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); assert.deepEqual(testState.workflowChecks, [{ - ns: "demo", worker: "api", allowCleanup: true, requestId: "rid-delete-no-cleanup", + ns: "demo", worker: "api", allowCleanup: true, requestId: "rid-delete-no-cleanup", lockToken: "lock-token", }]); assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old", requestId: "rid-delete-no-cleanup", @@ -1823,6 +1826,7 @@ test("version delete reports cleanup_queue_failed when data-plane cleanup enqueu version: "v1", allowCleanup: true, requestId: "rid-version-delete", + lockToken: "lock-token", }]); assert.equal(body.assets.queueHint, "failed"); assert.deepEqual(body.assets.warnings, [{ @@ -2035,6 +2039,7 @@ test("version delete reports queueHint none when no content cleanup is needed", version: "v1", allowCleanup: true, requestId: "rid-version-delete-no-cleanup", + lockToken: "lock-token", }]); assert.equal(body.assets.queueHint, "none"); assert.deepEqual(body.assets.warnings, []); diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 67a34809..6c8ccd0d 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -14,12 +14,14 @@ import { import { createFakeRedisSession, sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { readJsonResponse } from "../helpers/response-json.js"; import { realRuntimeInjectionSourcesUrl } from "../helpers/runtime-injection-sources.js"; +import { workflowDefinitionsUrl, workflowDefinitionRedisEval } from "../helpers/workflow-definitions.js"; const { libUrl: controlLibUrl, lifecycleIndexesUrl, sharedAuthRolesUrl, } = await compileControlGraph(); +const { WORKFLOW_DEFINITION_ADMISSION_SCRIPT } = await import(workflowDefinitionsUrl); /** @type {any} */ const CONTROL_DEPLOY_TEST_STATE = { @@ -331,6 +333,7 @@ const { commitWithWatch, handle } = await importControlHandler("control/handlers extraSharedSource: controlSharedExtraSource, replacements: { "control-lib": controlLibUrl, + "control-workflow-definitions": workflowDefinitionsUrl, "control-lifecycle-indexes": lifecycleIndexesUrl, "control-bundle": controlBundleUrl, "control-bindings": controlBindingsUrl, @@ -419,6 +422,7 @@ function makeSession() { nowMs: Date.now(), }; const session = createFakeRedisSession(fakeState, { + eval: workflowDefinitionRedisEval, onExecFailure() { state.execFailures = fakeState.execFailures; state.strings.set("d1:database-name:tenant-a:main", "d1_new"); @@ -1825,7 +1829,7 @@ test("commitWithWatch assigns stable workflow keys into bundle meta and wf:defs" ]); }); -test("commitWithWatch reads only workflow definitions declared by the new bundle", async () => { +test("commitWithWatch materializes only declared workflow definitions during quota checks", async () => { const duplicateKey = "wf_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; /** @type {unknown[][]} */ const workflowDefReads = []; @@ -1844,10 +1848,10 @@ test("commitWithWatch reads only workflow definitions declared by the new bundle /** @param {(s: ReturnType) => Promise} fn */ async session(fn) { const session = makeSession(); - const hMGet = session.hMGet.bind(session); - session.hMGet = async (key, fields) => { - workflowDefReads.push([key, [...fields]]); - return await hMGet(key, fields); + const evaluate = session.eval.bind(session); + session.eval = async (script, keys = [], args = []) => { + if (script === WORKFLOW_DEFINITION_ADMISSION_SCRIPT) workflowDefReads.push([keys[0], args.slice(2)]); + return await evaluate(script, keys, args); }; return await fn(session); }, diff --git a/tests/unit/control-handlers-workflows.test.js b/tests/unit/control-handlers-workflows.test.js index 2ef6963f..41ac8db4 100644 --- a/tests/unit/control-handlers-workflows.test.js +++ b/tests/unit/control-handlers-workflows.test.js @@ -12,6 +12,9 @@ import { compileControlGraph } from "../helpers/load-control-lib.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; +import { workflowDefinitionsUrl, workflowDefinitionRedisEval } from "../helpers/workflow-definitions.js"; +import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; const WORKFLOWS_HANDLER_GLOBAL = "__workflowsHandlerState"; @@ -19,6 +22,7 @@ const ACTIVE_WORKFLOW_KEY = `wf_${"1".repeat(32)}`; const NEW_WORKFLOW_KEY = `wf_${"2".repeat(32)}`; const RETIRED_WORKFLOW_KEY = `wf_${"3".repeat(32)}`; const { libUrl: productionControlLibUrl } = await compileControlGraph(); +const { WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT, WORKFLOW_ROUTE_PAGE_SCRIPT } = await import(workflowDefinitionsUrl); const workerContractUrl = repositoryFileUrl("shared/worker-contract.js"); const sharedNsPatternUrl = repositoryFileUrl("shared/ns-pattern.js"); @@ -29,6 +33,9 @@ const { handle } = await importControlHandler("control/handlers/workflows.js", { "shared-internal-auth": sharedInternalAuthUrl(), "shared-ns-pattern": sharedNsPatternUrl, "shared-worker-contract": workerContractUrl, + "control-workflow-definitions": workflowDefinitionsUrl, + "base64.js": repositoryFileUrl("shared/base64.js"), + "shared-utf8": repositoryFileUrl("shared/utf8.js"), }, }); @@ -42,6 +49,7 @@ function resetWorkflowsHandlerState() { }], }); const state = createControlHandlerState({ + redis: createFakeRedis(undefined, { eval: workflowDefinitionRedisEval }), env: { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }, }); const redis = /** @type {any} */ (state.redis); @@ -97,6 +105,7 @@ test("workflows handler lists active workflow definitions from bundle metadata", await assertJsonResponse(response, 200, { namespace: "demo", + cursor: null, workflows: [{ namespace: "demo", worker: "api", @@ -107,13 +116,8 @@ test("workflows handler lists active workflow definitions from bundle metadata", workflowKey: ACTIVE_WORKFLOW_KEY, }], }); - assert.deepEqual(state.redis.commands, [ - ["hGetAll", "routes:demo"], - ["hGetManyAndHGetAllMany", [ - ["worker:demo:api:v:2", "__meta__"], - ], ["wf:defs:demo:api"]], - ["hGetAll", "routes:demo"], - ]); + assert.equal(state.redis.commands.filter((/** @type {unknown[]} */ command) => command[0] === "eval" && command[1] === WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT).length, 1); + assert.equal(state.redis.commands.some((/** @type {unknown[]} */ command) => command[0] === "hGetAll"), false); assert.equal(sessionCalls, 1); assert.deepEqual(state.logs, [{ level: "info", @@ -122,6 +126,26 @@ test("workflows handler lists active workflow definitions from bundle metadata", }]); }); +test("definition lists sort names and prefer active metadata over retained definitions", async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set("worker:demo:api:v:2", { __meta__: JSON.stringify({ + workflows: [{ name: "zeta", binding: "ZETA", className: "CurrentWorkflow", workflowKey: ACTIVE_WORKFLOW_KEY }], + }) }); + state.redis.hashes.set("wf:defs:demo:api", { + zeta: JSON.stringify({ workflowKey: ACTIVE_WORKFLOW_KEY, className: "PreviousWorkflow" }), + alpha: JSON.stringify({ workflowKey: RETIRED_WORKFLOW_KEY, className: "RetiredWorkflow" }), + }); + const response = await handle({ method: "GET", url: new URL("http://control/ns/demo/workflows"), ns: "demo", subPath: [], requestId: "rid-definition-order" }); + await assertJsonResponse(response, 200, { + namespace: "demo", + cursor: null, + workflows: [ + { namespace: "demo", worker: "api", activeVersion: "v2", name: "alpha", binding: null, className: "RetiredWorkflow", workflowKey: RETIRED_WORKFLOW_KEY, retired: true }, + { namespace: "demo", worker: "api", activeVersion: "v2", name: "zeta", binding: "ZETA", className: "CurrentWorkflow", workflowKey: ACTIVE_WORKFLOW_KEY }, + ], + }); +}); + for (const [label, rawMeta] of [ ["missing", null], ["empty", ""], @@ -261,14 +285,13 @@ test("workflows handler retries a list snapshot split by whole-worker delete", a const state = resetWorkflowsHandlerState(); const redis = /** @type {any} */ (state.redis); configureWorkflowListSession(state, (session) => { - const originalRead = session.hGetManyAndHGetAllMany.bind(session); - session.hGetManyAndHGetAllMany = async ( - /** @type {Array<[string, string]>} */ pairs, - /** @type {string[]} */ keys - ) => { - const snapshot = await originalRead(pairs, keys); - redis.hashes.set("routes:demo", {}); - redis.hashes.set("worker:demo:api:v:2", {}); + const originalRead = session.eval.bind(session); + session.eval = async (/** @type {string} */ script, /** @type {string[]} */ keys, /** @type {unknown[]} */ args) => { + const snapshot = await originalRead(script, keys, args); + if (script === WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT) { + redis.hashes.set("routes:demo", {}); + redis.hashes.set("worker:demo:api:v:2", {}); + } return snapshot; }; }); @@ -284,6 +307,7 @@ test("workflows handler retries a list snapshot split by whole-worker delete", a await assertJsonResponse(response, 200, { namespace: "demo", workflows: [], + cursor: null, }); assert.deepEqual(state.logs, [{ level: "info", @@ -372,17 +396,17 @@ for (const [label, rawMeta] of [ redis.hashes.set("worker:demo:api:v:2", rawMeta == null ? {} : { "__meta__": rawMeta }); redis.hashes.set("worker:demo:billing:v:1", { "__meta__": emptyMeta }); redis.hashes.set("worker:demo:billing:v:2", { "__meta__": emptyMeta }); - const routeSnapshots = [ - { api: "v2", billing: "v1" }, - { api: "v2", billing: "v2" }, - { api: "v2", billing: "v2" }, - { api: "v2", billing: "v3" }, - ]; + redis.hashes.set("routes:demo", { api: "v2", billing: "v1" }); + let scans = 0; configureWorkflowListSession(state, (session) => { - session.hGetAll = async (/** @type {string} */ key) => { - redis.commands.push(["hGetAll", key]); - if (key === "routes:demo") return routeSnapshots.shift() ?? routeSnapshots.at(-1) ?? {}; - return redis.hashes.get(key) ?? {}; + const originalRead = session.eval.bind(session); + session.eval = async (/** @type {string} */ script, /** @type {string[]} */ keys, /** @type {unknown[]} */ args) => { + const snapshot = await originalRead(script, keys, args); + if (script === WORKFLOW_ROUTE_PAGE_SCRIPT) { + scans += 1; + redis.hashes.set("routes:demo", { api: "v2", billing: `v${scans + 1}` }); + } + return snapshot; }; }); @@ -401,7 +425,7 @@ for (const [label, rawMeta] of [ error: "corrupt_meta", message: "Internal error", }); - assert.equal(routeSnapshots.length, 2); + assert.equal(scans, 1); }); } @@ -410,17 +434,16 @@ test("workflows handler returns contention when list routes never stabilize", as const redis = /** @type {any} */ (state.redis); const meta = redis.hashes.get("worker:demo:api:v:2")["__meta__"]; redis.hashes.set("worker:demo:api:v:3", { "__meta__": meta }); - const routeSnapshots = [ - { api: "v2" }, - { api: "v3" }, - { api: "v3" }, - { api: "v4" }, - ]; + let snapshots = 0; configureWorkflowListSession(state, (session) => { - session.hGetAll = async (/** @type {string} */ key) => { - redis.commands.push(["hGetAll", key]); - if (key === "routes:demo") return routeSnapshots.shift() ?? {}; - return redis.hashes.get(key) ?? {}; + const originalRead = session.eval.bind(session); + session.eval = async (/** @type {string} */ script, /** @type {string[]} */ keys, /** @type {unknown[]} */ args) => { + const snapshot = await originalRead(script, keys, args); + if (script === WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT) { + snapshots += 1; + redis.hashes.set("routes:demo", { api: `v${snapshots + 2}` }); + } + return snapshot; }; }); @@ -437,7 +460,7 @@ test("workflows handler returns contention when list routes never stabilize", as message: "Internal error", namespace: "demo", }); - assert.equal(routeSnapshots.length, 0); + assert.equal(snapshots, 2); }); test("workflows handler does not combine active metadata with redeployed workflow defs", async () => { @@ -495,8 +518,9 @@ test("workflows handler lists empty namespaces without batch reads", async () => await assertJsonResponse(response, 200, { namespace: "demo", workflows: [], + cursor: null, }); - assert.deepEqual(state.redis.commands, [["hGetAll", "routes:demo"]]); + assert.equal(state.redis.commands.some((/** @type {unknown[]} */ command) => command[0] === "eval" && command[1] === WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT), false); assert.deepEqual(state.logs, [{ level: "info", event: "workflows_listed", @@ -504,13 +528,76 @@ test("workflows handler lists empty namespaces without batch reads", async () => }]); }); +test("definition pagination covers retired names and bounded worker batches without duplicates", async () => { + const state = resetWorkflowsHandlerState(); + const meta = state.redis.hashes.get("worker:demo:api:v:2").__meta__; + state.redis.hashes.set("wf:defs:demo:api", { + orders: JSON.stringify({ workflowKey: ACTIVE_WORKFLOW_KEY, className: "OrderWorkflow" }), + retired: JSON.stringify({ workflowKey: RETIRED_WORKFLOW_KEY, className: "OldOrderWorkflow" }), + }); + const routes = state.redis.hashes.get("routes:demo"); + for (let index = 0; index < 150; index += 1) { + const worker = `worker-${String(index).padStart(3, "0")}`; + routes[worker] = "v1"; + state.redis.hashes.set(`worker:demo:${worker}:v:1`, { __meta__: index === 149 ? meta : '{"workflows":[]}' }); + } + /** @type {string[]} */ + const seen = []; + let cursor = ""; + let pages = 0; + do { + const commandsBefore = state.redis.commands.length; + const url = new URL("http://control/ns/demo/workflows"); + url.searchParams.set("limit", "1"); + if (cursor) url.searchParams.set("cursor", cursor); + const response = await handle({ method: "GET", url, ns: "demo", subPath: [], requestId: "rid-definition-pages" }); + const body = await readJsonResponse(response, 200); + const snapshots = state.redis.commands.slice(commandsBefore).filter((/** @type {unknown[]} */ entry) => entry[0] === "eval" && entry[1] === WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT); + assert.ok(snapshots.length <= 16); + assert.ok(body.workflows.length <= 1); + for (const entry of body.workflows) seen.push(`${entry.worker}/${entry.name}/${entry.retired === true}`); + cursor = body.cursor; + pages += 1; + assert.ok(pages < 30); + } while (cursor !== null); + assert.deepEqual(seen, ["api/orders/false", "api/retired/true", "worker-149/orders/false"]); + assert.ok(pages > 3); +}); + +test("definition cursors reject snapshot drift rather than splicing changed definitions", async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set("wf:defs:demo:api", { + orders: JSON.stringify({ workflowKey: ACTIVE_WORKFLOW_KEY, className: "OrderWorkflow" }), + retired: JSON.stringify({ workflowKey: RETIRED_WORKFLOW_KEY, className: "OldOrderWorkflow" }), + }); + const first = await handle({ method: "GET", url: new URL("http://control/ns/demo/workflows?limit=1"), ns: "demo", subPath: [], requestId: "rid-first" }); + const cursor = (await readJsonResponse(first, 200)).cursor; + assert.equal(typeof cursor, "string"); + state.redis.hashes.get("wf:defs:demo:api").retired = JSON.stringify({ workflowKey: RETIRED_WORKFLOW_KEY, className: "ChangedWorkflow" }); + const url = new URL("http://control/ns/demo/workflows?limit=1"); + url.searchParams.set("cursor", cursor); + const changed = await handle({ method: "GET", url, ns: "demo", subPath: [], requestId: "rid-next" }); + assert.equal((await readJsonResponse(changed, 503)).error, "workflow_metadata_contention"); +}); + +test("definition query rejects invalid limits and cursors before Redis reads", async () => { + for (const query of ["limit=1001", "limit=0", "cursor=not-a-cursor", `cursor=${"x".repeat(2049)}`]) { + const state = resetWorkflowsHandlerState(); + const response = await handle({ method: "GET", url: new URL(`http://control/ns/demo/workflows?${query}`), ns: "demo", subPath: [], requestId: "rid-invalid-definition-page" }); + assert.equal((await readJsonResponse(response, 400)).error, "invalid_request"); + assert.equal(state.redis.commands.length, 0); + } +}); + test("workflows handler preserves metadata-unavailable error shape for batched reads", async () => { const state = resetWorkflowsHandlerState(); const redis = /** @type {any} */ (state.redis); redis.hashes.set("routes:demo", { api: "v2", billing: "v5" }); configureWorkflowListSession(state, (session) => { - session.hGetManyAndHGetAllMany = async () => { - throw new Error("redis unavailable"); + const originalRead = session.eval.bind(session); + session.eval = async (/** @type {string} */ script, /** @type {string[]} */ keys, /** @type {unknown[]} */ args) => { + if (script === WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT) throw new Error("redis unavailable"); + return originalRead(script, keys, args); }; }); @@ -557,8 +644,10 @@ test("workflows handler wraps combined metadata batch read failures", async () = const redis = /** @type {any} */ (state.redis); redis.hashes.set("routes:demo", { api: "v2", billing: "v5" }); configureWorkflowListSession(state, (session) => { - session.hGetManyAndHGetAllMany = async () => { - throw new Error("defs unavailable"); + const originalRead = session.eval.bind(session); + session.eval = async (/** @type {string} */ script, /** @type {string[]} */ keys, /** @type {unknown[]} */ args) => { + if (script === WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT) throw new Error("defs unavailable"); + return originalRead(script, keys, args); }; }); @@ -733,6 +822,140 @@ test("workflows handler forwards status includeSteps options", async () => { }]); }); +test("workflows handler preserves instance list cursor and payloads under a finite deadline", async () => { + const state = resetWorkflowsHandlerState(); + const result = { + instances: [{ id: "order-1", status: "completed", output: { text: "\u4e2d" }, error: null }], + cursor: "17", + }; + state.workflows = { + async fetch(/** @type {string} */ url, /** @type {RequestInit} */ init) { + assert.ok(init.signal instanceof AbortSignal); + assert.equal(new Headers(init.headers).get("x-wdl-internal-auth"), TEST_INTERNAL_AUTH_TOKEN); + assert.equal(new Headers(init.headers).get("x-request-id"), "rid-instances"); + assert.equal(url, "http://workflows/internal/workflows/instances"); + assert.deepEqual(parseJsonObjectRequestBody(init, "workflow list").options, { limit: 1000, cursor: "12" }); + return Response.json(result); + }, + }; + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances?limit=1000&cursor=12"), + ns: "demo", + subPath: ["api", "orders", "instances"], + requestId: "rid-instances", + }); + await assertJsonResponse(response, 200, result); +}); + +test("workflows instance list deadline cancels a stalled body and hides the failure", async () => { + const state = resetWorkflowsHandlerState(); + const controller = new AbortController(); + let cancelled = false; + state.workflows = { + async fetch(/** @type {string} */ _url, /** @type {RequestInit} */ init) { + assert.equal(init.signal, controller.signal); + return new Response(new ReadableStream({ + cancel() { cancelled = true; }, + })); + }, + }; + await withMockedProperty(AbortSignal, "timeout", (delay) => { + assert.equal(delay, 5_000); + return controller.signal; + }, async () => { + const pending = handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances"), + ns: "demo", + subPath: ["api", "orders", "instances"], + requestId: "rid-list-deadline", + }); + await new Promise((resolve) => setImmediate(resolve)); + controller.abort(new Error("private stalled Workflow transport")); + await assertJsonResponse(await pending, 503, { + error: "workflow_internal_dispatch_failed", + message: "Internal error", + }); + }); + assert.equal(cancelled, true); +}); + +test("workflows instance lists preserve the bounded JSON bytes without number re-encoding", async () => { + const state = resetWorkflowsHandlerState(); + const wire = '{"instances":[{"id":"order-1","status":"completed","output":1e20,"error":null}],"cursor":null}'; + state.workflows = { fetch: async () => new Response(wire) }; + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances"), + ns: "demo", + subPath: ["api", "orders", "instances"], + requestId: "rid-list-wire", + }); + assert.equal(response.status, 200); + assert.equal(response.headers.get("content-type"), "application/json"); + assert.equal(await response.text(), wire); +}); + +test("workflows instance list preserves sanitized backend errors", async () => { + const state = resetWorkflowsHandlerState(); + state.workflows = { fetch: async () => Response.json({ + error: "workflow_payload_missing", message: "private missing payload reference", + }, { status: 500 }) }; + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances"), + ns: "demo", + subPath: ["api", "orders", "instances"], + requestId: "rid-list-payload-missing", + }); + await assertJsonResponse(response, 500, { + error: "workflow_payload_missing", + message: "Internal error", + upstream_status: 500, + }); +}); + +test("definition snapshots retry route drift before reading stale metadata", async () => { + const state = resetWorkflowsHandlerState(); + const redis = state.redis; + const meta = redis.hashes.get("worker:demo:api:v:2").__meta__; + redis.hashes.set("worker:demo:api:v:3", { __meta__: meta }); + let snapshots = 0; + configureWorkflowListSession(state, (session) => { + const originalRead = session.eval.bind(session); + session.eval = async (/** @type {string} */ script, /** @type {string[]} */ keys, /** @type {unknown[]} */ args) => { + if (script === WORKFLOW_DEFINITION_SNAPSHOT_SCRIPT && snapshots++ === 0) { + redis.hashes.set("routes:demo", { api: "v3" }); + redis.hashes.set("worker:demo:api:v:2", { __meta__: "stale invalid metadata" }); + } + return originalRead(script, keys, args); + }; + }); + const response = await handle({ method: "GET", url: new URL("http://control/ns/demo/workflows"), ns: "demo", subPath: [], requestId: "rid-atomic-snapshot" }); + const body = await readJsonResponse(response, 200); + assert.equal(body.workflows.length, 1); + assert.equal(body.workflows[0].activeVersion, "v3"); + assert.equal(snapshots, 2); +}); + +test("a full definition page needs at most one snapshot call per worker plus two route scans", async () => { + const state = resetWorkflowsHandlerState(); + const meta = state.redis.hashes.get("worker:demo:api:v:2").__meta__; + const routes = state.redis.hashes.get("routes:demo"); + for (let index = 0; index < 15; index += 1) { + const worker = `worker-${index}`; + routes[worker] = "v1"; + state.redis.hashes.set(`worker:demo:${worker}:v:1`, { __meta__: meta }); + } + const response = await handle({ method: "GET", url: new URL("http://control/ns/demo/workflows"), ns: "demo", subPath: [], requestId: "rid-snapshot-budget" }); + const body = await readJsonResponse(response, 200); + assert.equal(body.workflows.length, 16); + assert.equal(body.cursor, null); + const reads = state.redis.commands.filter((/** @type {unknown[]} */ command) => ["eval", "hGet", "hGetAll"].includes(String(command[0]))); + assert.ok(reads.length <= 18, `definition page used ${reads.length} Redis calls`); +}); + test("workflows handler fails closed when workflows backend is unavailable", async () => { const state = resetWorkflowsHandlerState(); state.workflows = null; diff --git a/tests/unit/control-index.test.js b/tests/unit/control-index.test.js index 047294ab..217fb879 100644 --- a/tests/unit/control-index.test.js +++ b/tests/unit/control-index.test.js @@ -168,7 +168,7 @@ test("control whoami uses sanitized forwarded proto for public URL hints", async const body = await readJsonResponse(response, 200); assert.equal(body.ok, true); assert.equal(body.platformVersion, "wdl.20990101.2"); - assert.equal(body.minCliVersion, "1.8.0"); + assert.equal(body.minCliVersion, "1.9.0"); assert.deepEqual(body.urls, { control: "https://control.example", namespace: "https://tenant-a.workers.example", diff --git a/tests/unit/control-shared.test.js b/tests/unit/control-shared.test.js index 7da05017..ddef2e05 100644 --- a/tests/unit/control-shared.test.js +++ b/tests/unit/control-shared.test.js @@ -5,14 +5,14 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { moduleDataUrl } from "../helpers/load-shared-module.js"; +import { moduleDataUrl, readRepositoryJson } from "../helpers/load-shared-module.js"; import { compileControlSharedGraph } from "../helpers/load-control-shared.js"; import { createFakeRedis, createFakeRedisState, sharedRedisStubUrl, } from "../helpers/mocks/fake-redis.js"; -import { installMockProperty } from "../helpers/mock-global.js"; +import { installMockProperty, withMockedProperty } from "../helpers/mock-global.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; import { assertJsonResponse } from "../helpers/response-json.js"; @@ -35,7 +35,16 @@ const { controlSharedUrl, controlWorkflowsClientUrl } = compileControlSharedGrap sharedAuthRolesUrl, sharedQueueKeysUrl, }); -const { postWorkflowsInternalRequest } = await import(controlWorkflowsClientUrl); +const { + WORKFLOWS_INTERNAL_TIMEOUT_MS, + MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES, + WORKFLOW_LIFECYCLE_MAX_PAGES, + WORKFLOW_LIFECYCLE_TIMEOUT_MS, + WORKFLOW_LIFECYCLE_RESPONSE_MAX_BYTES, + postWorkflowsInternalRequest, + readWorkflowInstancesResponse, + readWorkflowLifecycleResponse, +} = await import(controlWorkflowsClientUrl); // Use the same stub constructor imported by control/shared.js so // runOptimistic's `instanceof WatchError` check observes the test error. const { WatchError: ControlSharedWatchError } = await import(sharedRedisUrl); @@ -64,11 +73,13 @@ const { */ function restoreControlSharedStateAfter(t) { const previous = { + redis: state.redis, env: state.env, log: state.log, workflows: state.workflows, }; t.after(() => { + state.redis = previous.redis; state.env = previous.env; state.log = previous.log; state.workflows = previous.workflows; @@ -639,7 +650,7 @@ test("assertWorkflowDeleteAllowed fails malformed successful workflow response a const abort = /** @type {InstanceType} */ (err); assert.equal(abort.status, 503); assert.equal(abort.code, "workflow_internal_dispatch_failed"); - assert.equal(abort.details.message, "Workflow lifecycle check returned an invalid response"); + assert.equal(abort.details.message, "Workflow lifecycle check failed"); return true; } ); @@ -752,7 +763,8 @@ test("shared workflows calls preserve endpoint-specific timeout behavior", async const endpoint = String(url); if (endpoint.endsWith("/lifecycle/check-delete")) { assert.equal(new Headers(init?.headers).get("x-request-id"), "rid-lifecycle"); - assert.equal(init?.signal, undefined); + assert.ok(init?.signal instanceof AbortSignal); + fetchSignals.push(init.signal); requestBodies.lifecycle = parseJsonObjectRequestBody(init, "workflow lifecycle request body"); return Response.json({ allowed: true }); } @@ -781,8 +793,8 @@ test("shared workflows calls preserve endpoint-specific timeout behavior", async restoreTimeout(); } - assert.deepEqual(timeoutMs, [5_000]); - assert.equal(timeoutSignals.length, 1); + assert.deepEqual(timeoutMs, [5_000, 5_000]); + assert.equal(timeoutSignals.length, 2); assert.deepEqual(timeoutSignals, fetchSignals); assert.deepEqual(requestBodies, { lifecycle: { ns: "demo", worker: "api", version: "v2" }, @@ -790,6 +802,318 @@ test("shared workflows calls preserve endpoint-specific timeout behavior", async }); }); +const lifecycleContract = /** @type {{ limits: { responseBytesMax: number, controlPagesMax: number, controlTimeoutMs: number }, request: Record, responses: { complete: Record, blocked: Record, continuation: Record, rescanRequired: Record } }} */ ( + readRepositoryJson("tests/fixtures/workflow-lifecycle-check.json") +); + +test("Workflow lifecycle reader limits match the cross-language fixture", async () => { + assert.equal(WORKFLOW_LIFECYCLE_MAX_PAGES, lifecycleContract.limits.controlPagesMax); + assert.equal(WORKFLOW_LIFECYCLE_TIMEOUT_MS, lifecycleContract.limits.controlTimeoutMs); + assert.equal(WORKFLOW_LIFECYCLE_RESPONSE_MAX_BYTES, lifecycleContract.limits.responseBytesMax); + for (const body of Object.values(lifecycleContract.responses)) { + assert.deepEqual(await readWorkflowLifecycleResponse(Response.json(body)), body); + } +}); + +test("Workflow lifecycle pages resume internally and renew the same delete lock", async (t) => { + restoreControlSharedStateAfter(t); + state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; + const redis = createFakeRedis(); + state.redis = redis; + const token = await acquireDeleteLock(redis, "demo", "api", "whole"); + assert.ok(token); + /** @type {Record[]} */ + const bodies = []; + state.workflows = { + async fetch(/** @type {string} */ _url, /** @type {RequestInit} */ init) { + bodies.push(parseJsonObjectRequestBody(init, "lifecycle page")); + return Response.json(bodies.length === 1 ? lifecycleContract.responses.continuation : lifecycleContract.responses.complete); + }, + }; + await assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", version: "v2", allowCleanup: true, lockToken: token }); + assert.deepEqual(bodies, [ + { ns: "demo", worker: "api", version: "v2", allowCleanup: true }, + lifecycleContract.request, + ]); + assert.equal(redis.commands.filter((command) => command[0] === "set" && /** @type {{ifeq?: string}} */ (command[3] ?? {}).ifeq === token).length, 2); +}); + +test("Workflow lifecycle pagination stops when the delete lock is lost", async (t) => { + restoreControlSharedStateAfter(t); + state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; + const redis = createFakeRedis(); + state.redis = redis; + const token = await acquireDeleteLock(redis, "demo", "api", "whole"); + assert.ok(token); + let calls = 0; + state.workflows = { + async fetch() { + calls += 1; + await releaseDeleteLock(redis, "demo", "api", token); + return Response.json(lifecycleContract.responses.continuation); + }, + }; + await assert.rejects( + () => assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", allowCleanup: true, lockToken: token }), + (error) => error instanceof ControlAbort && /** @type {{code?: string}} */ (error).code === "deleting", + ); + assert.equal(calls, 1); +}); + +test("Workflow lifecycle rescan responses require a new request instead of reporting active instances", async (t) => { + restoreControlSharedStateAfter(t); + state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; + for (const afterContinuation of [false, true]) { + let calls = 0; + state.workflows = { + async fetch() { + calls += 1; + return Response.json(afterContinuation && calls === 1 + ? lifecycleContract.responses.continuation + : lifecycleContract.responses.rescanRequired); + }, + }; + await assert.rejects( + () => assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", allowCleanup: true }), + (error) => error instanceof ControlAbort && + /** @type {{status?: number, code?: string}} */ (error).status === 503 && + /** @type {{code?: string}} */ (error).code === "workflow_lifecycle_check_incomplete", + ); + assert.equal(calls, afterContinuation ? 2 : 1); + } +}); + +for (const phase of ["fetch", "body", "materialization"]) { + test(`Workflow lifecycle total deadline during ${phase} returns incomplete`, async (t) => { + restoreControlSharedStateAfter(t); + state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; + let now = 1_000; + const deadline = now + WORKFLOW_LIFECYCLE_TIMEOUT_MS; + let calls = 0; + let cancelled = false; + /** @type {number[]} */ + const timeouts = []; + /** @type {AbortController[]} */ + const controllers = []; + /** @type {Array<{level: string, event: string}>} */ + const logs = []; + state.log = (/** @type {string} */ level, /** @type {string} */ event) => logs.push({ level, event }); + const expire = () => { + now = deadline; + const controller = controllers.at(-1); + assert.ok(controller); + controller.abort(new DOMException("deadline expired", "TimeoutError")); + }; + state.workflows = { + async fetch(/** @type {string} */ _url, /** @type {RequestInit} */ init) { + calls += 1; + if (calls < 3) { + now += WORKFLOWS_INTERNAL_TIMEOUT_MS - 1; + return Response.json({ ...lifecycleContract.responses.continuation, cursor: String(calls) }); + } + if (phase === "body") { + return new Response(new ReadableStream({ + pull() { expire(); }, + cancel() { cancelled = true; }, + }, { highWaterMark: 0 })); + } + now = deadline; + if (phase === "fetch") { + expire(); + throw init.signal?.reason; + } + return Response.json(lifecycleContract.responses.complete); + }, + }; + await withMockedProperty(Date, "now", () => now, async () => { + await withMockedProperty(AbortSignal, "timeout", (ms) => { + timeouts.push(ms); + const controller = new AbortController(); + controllers.push(controller); + return controller.signal; + }, async () => { + await assert.rejects( + () => assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", allowCleanup: true }), + (error) => error instanceof ControlAbort && + /** @type {{status?: number, code?: string}} */ (error).status === 503 && + /** @type {{code?: string}} */ (error).code === "workflow_lifecycle_check_incomplete", + ); + }); + }); + assert.equal(calls, 3); + assert.deepEqual(timeouts, [WORKFLOWS_INTERNAL_TIMEOUT_MS, WORKFLOWS_INTERNAL_TIMEOUT_MS, 2]); + assert.equal(cancelled, phase === "body"); + assert.deepEqual(logs, [{ level: "warn", event: "workflow_lifecycle_check_failed" }]); + }); +} + +test("Workflow lifecycle preserves per-call timeouts and backend errors", async (t) => { + restoreControlSharedStateAfter(t); + state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; + for (const scenario of ["per-call timeout", "late success", "late transport failure", "malformed reply"]) { + let now = 1_000; + const controller = new AbortController(); + /** @type {string[]} */ + const levels = []; + state.log = (/** @type {string} */ level) => levels.push(level); + state.workflows = { async fetch() { + if (scenario === "per-call timeout") { + now += WORKFLOWS_INTERNAL_TIMEOUT_MS; + controller.abort(new DOMException("per-call timeout", "TimeoutError")); + throw controller.signal.reason; + } + now += WORKFLOW_LIFECYCLE_TIMEOUT_MS; + if (scenario === "late success") return Response.json(lifecycleContract.responses.complete); + if (scenario === "malformed reply") return new Response("{broken"); + throw new Error("backend connection failed"); + } }; + await withMockedProperty(Date, "now", () => now, async () => { + await withMockedProperty(AbortSignal, "timeout", () => controller.signal, async () => { + await assert.rejects( + () => assertWorkflowDeleteAllowed({ ns: "demo", worker: "api" }), + (error) => error instanceof ControlAbort && + /** @type {{code?: string}} */ (error).code === "workflow_internal_dispatch_failed", + ); + }); + }); + assert.deepEqual(levels, ["error"]); + } +}); + +for (const phase of ["fetch", "body"]) { + test(`Workflow lifecycle preserves a per-call timeout after delayed ${phase} rejection`, async (t) => { + restoreControlSharedStateAfter(t); + state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; + const controller = new AbortController(); + const started = Promise.withResolvers(); + let now = 1_000; + let calls = 0; + /** @type {string[]} */ + const levels = []; + state.log = (/** @type {string} */ level) => levels.push(level); + state.workflows = { async fetch() { + calls += 1; + if (phase === "body") { + return new Response(new ReadableStream({ + pull() { started.resolve(undefined); }, + }, { highWaterMark: 0 })); + } + return await new Promise((_, reject) => { + controller.signal.addEventListener("abort", () => reject(controller.signal.reason), { once: true }); + started.resolve(undefined); + }); + } }; + await withMockedProperty(Date, "now", () => now, async () => { + await withMockedProperty(AbortSignal, "timeout", (ms) => { + assert.equal(ms, WORKFLOWS_INTERNAL_TIMEOUT_MS); + return controller.signal; + }, async () => { + const pending = assertWorkflowDeleteAllowed({ ns: "demo", worker: "api" }); + const rejected = assert.rejects(pending, (error) => error instanceof ControlAbort && + /** @type {{code?: string}} */ (error).code === "workflow_internal_dispatch_failed"); + await started.promise; + now = 1_000 + WORKFLOWS_INTERNAL_TIMEOUT_MS; + controller.abort(new DOMException("per-call timeout", "TimeoutError")); + // Deliver the rejection only after both deadlines have passed. + now = 1_000 + WORKFLOW_LIFECYCLE_TIMEOUT_MS + 1; + await rejected; + }); + }); + assert.equal(calls, 1); + assert.deepEqual(levels, ["error"]); + }); +} + +test("Workflow lifecycle bounds a hung renewal by the remaining check budget", async (t) => { + restoreControlSharedStateAfter(t); + state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; + const redis = createFakeRedis(); + state.redis = redis; + const token = await acquireDeleteLock(redis, "demo", "api", "whole"); + assert.ok(token); + const set = redis.set.bind(redis); + const renewalStarted = Promise.withResolvers(); + const lateRenewal = Promise.withResolvers(); + let now = 1_000; + let renewals = 0; + let backendCalls = 0; + /** @type {Array<{callback: () => void, delay: number, cleared: boolean}>} */ + const timers = []; + const setTimeoutMock = /** @type {typeof setTimeout} */ (( + /** @type {() => void} */ callback, /** @type {number} */ delay + ) => { + const timer = { callback, delay, cleared: false }; + timers.push(timer); + return /** @type {ReturnType} */ (/** @type {unknown} */ (timer)); + }); + const clearTimeoutMock = /** @type {typeof clearTimeout} */ ((timer) => { + /** @type {{cleared: boolean}} */ (/** @type {unknown} */ (timer)).cleared = true; + }); + state.workflows = { + async fetch() { + backendCalls += 1; + now += 2_000; + return Response.json(lifecycleContract.responses.continuation); + }, + }; + await withMockedProperty(Date, "now", () => now, async () => { + await withMockedProperty(globalThis, "setTimeout", setTimeoutMock, async () => { + await withMockedProperty(globalThis, "clearTimeout", clearTimeoutMock, async () => { + await withMockedProperty(redis, "set", (/** @type {Parameters} */ ...args) => { + renewals += 1; + if (renewals === 1) return set(...args); + renewalStarted.resolve(undefined); + return lateRenewal.promise; + }, async () => { + const check = assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", lockToken: token }); + const rejected = assert.rejects(check, (error) => error instanceof ControlAbort && + /** @type {{code?: string}} */ (error).code === "workflow_lifecycle_check_incomplete"); + await renewalStarted.promise; + assert.equal(timers[0].cleared, true); + assert.equal(timers[1].delay, lifecycleContract.limits.controlTimeoutMs - 2_000); + now = 1_000 + lifecycleContract.limits.controlTimeoutMs; + timers[1].callback(); + await rejected; + assert.equal(timers[1].cleared, true); + assert.equal(backendCalls, 1); + lateRenewal.reject(new Error("late Redis rejection")); + await Promise.resolve(); + }); + }); + }); + }); +}); + +test("Workflow lifecycle continuation cannot turn an incomplete or blocked scan into permission", async (t) => { + restoreControlSharedStateAfter(t); + state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; + for (const scenario of ["budget", "blocked", "stalled", "contradictory"]) { + let calls = 0; + state.workflows = { + async fetch() { + calls += 1; + if (scenario === "blocked" && calls === 2) return Response.json(lifecycleContract.responses.blocked); + if (scenario === "contradictory") return Response.json({ ...lifecycleContract.responses.complete, cursor: "128" }); + return Response.json({ ...lifecycleContract.responses.continuation, cursor: scenario === "stalled" ? "128" : String(calls) }); + }, + }; + const expected = scenario === "budget" ? "workflow_lifecycle_check_incomplete" + : scenario === "blocked" ? "workflow_instances_active" : "workflow_internal_dispatch_failed"; + await assert.rejects(() => assertWorkflowDeleteAllowed({ ns: "demo", worker: "api" }), + (error) => error instanceof ControlAbort && /** @type {{code?: string}} */ (error).code === expected); + assert.equal(calls, scenario === "budget" ? WORKFLOW_LIFECYCLE_MAX_PAGES : scenario === "contradictory" ? 1 : 2); + } +}); + +test("Workflow lifecycle reader rejects oversized replies without draining the body", async () => { + let cancelled = false; + const response = new Response(new ReadableStream({ cancel() { cancelled = true; } }), { + headers: { "content-length": String(WORKFLOW_LIFECYCLE_RESPONSE_MAX_BYTES + 1) }, + }); + await assert.rejects(() => readWorkflowLifecycleResponse(response), /exceeds/); + assert.equal(cancelled, true); +}); + test("DO alarm cleanup rejects malformed or oversized success envelopes", async (t) => { restoreControlSharedStateAfter(t); state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; @@ -825,24 +1149,133 @@ test("DO alarm cleanup rejects malformed or oversized success envelopes", async test("workflows transport requires an explicit timeout selection at runtime", async () => { let fetchCalls = 0; - await assert.rejects( - postWorkflowsInternalRequest({ - workflows: { - async fetch() { - fetchCalls += 1; - return Response.json({ ok: true }); + for (const deadlineMs of [undefined, Date.now() + 5_000, Date.now() - 1]) { + await assert.rejects( + postWorkflowsInternalRequest({ + workflows: { + async fetch() { + fetchCalls += 1; + return Response.json({ ok: true }); + }, }, + headers: () => ({ "content-type": "application/json" }), + endpoint: "workflows/test", + body: {}, + logEvent: "workflow_test_failed", + timeoutMs: /** @type {any} */ (undefined), + deadlineMs, + makeError: (/** @type {import("../../control/workflows-client.js").WorkflowTransportFailure} */ failure) => new Error(failure), + }), + /request_failed/ + ); + } + assert.equal(fetchCalls, 0); +}); + +test("workflows transport handles explicit null and tied caps without trusting upstream timeouts", async () => { + for (const { timeoutMs, outcome } of [null, 1_000].flatMap((timeoutMs) => + ["success", "upstream timeout", "clock expiry"].map((outcome) => ({ timeoutMs, outcome })))) { + let now = 1_000; + const controller = new AbortController(); + await withMockedProperty(Date, "now", () => now, async () => { + await withMockedProperty(AbortSignal, "timeout", (ms) => { + assert.equal(ms, 1_000); + return controller.signal; + }, async () => { + const pending = postWorkflowsInternalRequest({ + workflows: { async fetch() { + if (outcome !== "success") now = 3_000; + if (outcome === "upstream timeout") { + throw new DOMException("upstream timeout", "TimeoutError"); + } + return Response.json({ ok: true }); + } }, + headers: () => ({}), + endpoint: "workflows/test", + body: {}, + logEvent: "workflow_test_failed", + timeoutMs, + deadlineMs: 2_000, + makeError: (/** @type {import("../../control/workflows-client.js").WorkflowTransportFailure} */ failure) => new Error(failure), + }); + if (outcome === "upstream timeout") await assert.rejects(pending, /request_failed/); + else if (outcome === "clock expiry") await assert.rejects(pending, /deadline/); + else assert.deepEqual((await pending).body, { ok: true }); + }); + }); + } +}); + +test("Workflow instances reader shares its byte ceiling with the Rust fixture", () => { + const contract = /** @type {{ instancesResponseBytesMax: number }} */ ( + readRepositoryJson("tests/fixtures/workflow-limits.json") + ); + assert.equal(MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES, contract.instancesResponseBytesMax); +}); + +for (const character of ["x", "\u4e2d"]) { + test(`Workflow instances reader accepts exact-limit UTF-8 (${character})`, async () => { + const empty = JSON.stringify({ instances: [{ output: "" }], cursor: null }); + const available = MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES - Buffer.byteLength(empty); + const width = Buffer.byteLength(character); + const output = character.repeat(Math.floor(available / width)) + "x".repeat(available % width); + const body = JSON.stringify({ instances: [{ output }], cursor: null }); + assert.equal(Buffer.byteLength(body), MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES); + const response = await readWorkflowInstancesResponse(new Response(body)); + assert.equal(response.body.instances[0].output, output); + assert.equal(response.body.cursor, null); + assert.equal(response.bytes.byteLength, MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES); + }); +} + +test("Workflow instances reader cancels an oversized declared body without waiting for cancel", async () => { + let cancelled = false; + const response = new Response(new ReadableStream({ + cancel() { cancelled = true; return new Promise(() => {}); }, + }), { headers: { "content-length": String(MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES + 1) } }); + await assert.rejects(() => readWorkflowInstancesResponse(response), /exceeds/); + assert.equal(cancelled, true); +}); + +for (const headers of /** @type {HeadersInit[]} */ ([{}, { "content-encoding": "gzip", "content-length": "1" }])) { + test(`Workflow instances reader caps streamed bytes with ${JSON.stringify(headers)}`, async () => { + let cancelled = false; + const response = new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(MAX_WORKFLOW_INSTANCES_RESPONSE_BYTES)); + controller.enqueue(new Uint8Array(1)); }, - headers: () => ({ "content-type": "application/json" }), - endpoint: "workflows/test", + cancel() { cancelled = true; }, + }), { headers }); + await assert.rejects(() => readWorkflowInstancesResponse(response), /exceeds/); + assert.equal(cancelled, true); + }); +} + +test("Workflow instances reader rejects malformed JSON and UTF-8", async () => { + for (const body of ["not-json", new Uint8Array([0xff])]) { + await assert.rejects(() => readWorkflowInstancesResponse(new Response(body))); + } +}); + +test("finite Workflow transport deadlines include synchronous response materialization", async () => { + let now = 1_000; + await withMockedProperty(Date, "now", () => now, async () => { + await assert.rejects(() => postWorkflowsInternalRequest({ + workflows: { fetch: async () => Response.json({ instances: [], cursor: null }) }, + headers: () => ({}), + endpoint: "workflows/instances", body: {}, logEvent: "workflow_test_failed", - timeoutMs: /** @type {any} */ (undefined), - makeError: (/** @type {"unavailable" | "request_failed"} */ failure) => new Error(failure), - }), - /request_failed/ - ); - assert.equal(fetchCalls, 0); + timeoutMs: 5_000, + readBody: async (/** @type {Response} */ response) => { + const body = await response.json(); + now += 5_000; + return body; + }, + makeError: (/** @type {string} */ failure) => new Error(failure), + }), /request_failed/); + }); }); test("codedErrorResponse preserves semantic status/code with a fallback code", async () => { diff --git a/tests/unit/integration-test-plan.test.js b/tests/unit/integration-test-plan.test.js index 82030567..6f642948 100644 --- a/tests/unit/integration-test-plan.test.js +++ b/tests/unit/integration-test-plan.test.js @@ -38,16 +38,16 @@ test("prioritizeDefaultFiles runs configured slow files first and preserves the "d1-storage-shared-localdisk.test.js", "d1-ownership-multi-runtime.test.js", "durable-objects-alarms.test.js", + "cron-triggers.test.js", "log-tail.test.js", "durable-objects-websocket.test.js", "workflows-runtime-scheduler.test.js", "queues-retry-and-delay.test.js", - "cron-triggers.test.js", - "queues-orphan-and-control.test.js", - "queues-delivery.test.js", "workflows-runtime-core.test.js", - "queues-batch-and-isolation.test.js", + "queues-delivery.test.js", + "queues-orphan-and-control.test.js", "workflows-runtime-retention.test.js", + "queues-batch-and-isolation.test.js", "z.test.js", "a.test.js", ] diff --git a/tests/unit/runtime-dispatch-workflows.test.js b/tests/unit/runtime-dispatch-workflows.test.js index b6fe898d..6c4a2eca 100644 --- a/tests/unit/runtime-dispatch-workflows.test.js +++ b/tests/unit/runtime-dispatch-workflows.test.js @@ -1763,6 +1763,81 @@ test("handleWorkflowRunDispatch does not replay a completed sleep as step.do", a ]); }); +test("step controller synchronously releases and cancels pending replay reads on closure", async () => { + const reading = Promise.withResolvers(); + let cancelled = false; + let callbacks = 0; + /** @type {string[]} */ + const calls = []; + const controller = createStepController({ + ns: "demo", worker: "shop", frozenVersion: "v1", workflowName: "orders", + workflowKey: "wf_pending_read", className: "OrderWorkflow", instanceId: "pending", + generation: 1, runToken: "run-1", createdAtMs: 1, dispatchDeadlineMs: Date.now() + 60_000, + }, { + async fetch(/** @type {string} */ url) { + calls.push(url); + return new Response(new ReadableStream({ + pull() { reading.resolve(undefined); }, + cancel() { cancelled = true; }, + }, { highWaterMark: 0 }), { headers: { "content-length": String(1024 * 1024) } }); + }, + }); + const pending = controller.facade.do("pending", async () => { callbacks += 1; }); + const rejection = assert.rejects(pending, (error) => error instanceof Error && error.name === "workflow_invalid_step"); + await reading.promise; + const probe = runtimeDispatchWorkflowReplayCache.createWorkflowReplayReadLease(); + const max = runtimeDispatchWorkflowReplayCache.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES; + try { + assert.throws(() => probe.reserve(max), runtimeDispatchWorkflowReplayCache.WorkflowReplayCapacityError); + controller.closeForRunReturn(); + assert.equal(cancelled, true); + assert.doesNotThrow(() => probe.reserve(max), "release must not wait for the read finally"); + controller.closeForRunReturn(); + await rejection; + assert.equal(callbacks, 0); + assert.equal(calls.length, 1); + assert.ok(calls[0].endsWith("/replay-steps")); + } finally { + controller.closeForRunReturn(); + probe.release(); + } +}); + +test("replay capacity pressure is retryable without a fresh claim and recovers after release", async () => { + const capacity = runtimeDispatchWorkflowReplayCache.createWorkflowReplayReadLease(); + capacity.reserve(runtimeDispatchWorkflowReplayCache.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES); + let callbackCalls = 0; + const run = { + ns: "demo", worker: "shop", frozenVersion: "v1", workflowName: "orders", + workflowKey: "wf_capacity", className: "OrderWorkflow", instanceId: "capacity", + generation: 1, createdAtMs: 12345, runToken: "run-1", event: { payload: {} }, + }; + const backend = makeWorkflowBackend(async () => { + throw new Error("capacity refusal must not call claim or commit"); + }, { + replayPage: () => ({ + steps: [{ ordinal: 0, name: "cached", nameCount: 1, dependencies: [], kind: "do", config: "null", status: "completed", output: "stored" }], + nextOrdinal: 1, done: true, + }), + }); + const stub = makeStub({ entrypoints: { OrderWorkflow: { + async run(/** @type {unknown} */ _event, /** @type {any} */ step) { + return await step.do("cached", async () => { callbackCalls += 1; return "fresh"; }); + }, + } } }); + try { + const failed = await handleWorkflowRunDispatch({ run, scope: makeScope(), env: workflowEnv(backend), stub }); + assert.equal((await readJsonResponse(failed, 503)).error, "workflow_backend_unavailable"); + } finally { + capacity.release(); + } + const completed = await handleWorkflowRunDispatch({ run: { ...run, runToken: "run-2" }, scope: makeScope(), env: workflowEnv(backend), stub }); + assert.equal((await readJsonResponse(completed, 200)).output, "stored"); + assert.equal(callbackCalls, 0); + assert.equal(backend.calls.length, 2); + assert.ok(backend.calls.every((call) => call.url.endsWith("/replay-steps"))); +}); + test("handleWorkflowRunDispatch keeps an evicted in-flight replay page locally usable", async () => { const scope = makeScope(); const stepCount = 40; diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 0b9e1988..5830beaa 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -865,6 +865,53 @@ test("offline Workflow step conversion shares legacy shapes with service integra ]) assert.match(readRepoFile(reader), /workflow-schema2-steps\.json/, reader); }); +test("Workflow payload and instance-list limits share a cross-language fixture", () => { + const contract = /** @type {Record} */ ( + readRepositoryJson("tests/fixtures/workflow-limits.json") + ); + assert.deepEqual(Object.keys(contract).toSorted(), [ + "backendRequestBytesMax", + "createBatchMax", + "instancesResponseBytesMax", + "jsonContainerDepthMax", + "paramsBytesMax", + "payloadTooLargeCode", + "rejectLoneSurrogates", + "resultBytesMax", + ]); + for (const key of ["backendRequestBytesMax", "createBatchMax", "instancesResponseBytesMax", "jsonContainerDepthMax", "paramsBytesMax", "resultBytesMax"]) { + assert.ok(Number.isSafeInteger(contract[key]) && Number(contract[key]) > 0, key); + } + assert.equal(typeof contract.payloadTooLargeCode, "string"); + assert.equal(contract.rejectLoneSurrogates, true); + for (const reader of [ + "rust/workflows/src/api/limits.rs", + "tests/unit/control-shared.test.js", + "tests/unit/runtime-workflows-client.test.js", + "tests/unit/runtime-dispatch-workflows.test.js", + "tests/integration/workflows-runtime-retention.test.js", + ]) { + assert.match(readRepoFile(reader), /workflow-limits\.json/, reader); + } +}); + +test("Workflow lifecycle continuation is shared by Rust and Control", () => { + const fixture = "tests/fixtures/workflow-lifecycle-check.json"; + const contract = /** @type {{ request: Record, responses: Record, limits: Record }} */ (readRepositoryJson(fixture)); + assert.deepEqual(Object.keys(contract.responses).toSorted(), ["blocked", "complete", "continuation", "rescanRequired"]); + assert.equal(contract.responses.complete.allowed, true); + assert.equal(contract.responses.blocked.allowed, false); + assert.equal(contract.responses.continuation.allowed, false); + assert.equal(typeof contract.responses.continuation.cursor, "string"); + assert.equal(contract.responses.rescanRequired.allowed, false); + assert.deepEqual(contract.responses.rescanRequired.blockers, []); + assert.equal(Object.hasOwn(contract.responses.rescanRequired, "cursor"), false); + for (const limit of Object.values(contract.limits)) assert.ok(Number.isSafeInteger(limit) && limit > 0); + for (const reader of ["rust/workflows/src/api/lifecycle/cleanup.rs", "tests/unit/control-shared.test.js"]) { + assert.match(readRepoFile(reader), /workflow-lifecycle-check\.json/, reader); + } +}); + test("Workflow Runtime run requests share one JS and Rust fixture", () => { const fixture = "tests/fixtures/workflow-runtime-request.json"; const contract = /** @type {{ run: Record }} */ ( @@ -3207,6 +3254,19 @@ test("D1 and DO deployments keep explicit runtime memory ceilings", () => { } }); +test("Terraform system-runtime exposes operator-controlled capacity", () => { + const name = "system_runtime_desired_count"; + const rootVars = withoutLineComments(readRepoFile("terraform/variables.tf")); + const moduleVars = withoutLineComments(readRepoFile("terraform/modules/compute/variables.tf")); + const main = withoutLineComments(readRepoFile("terraform/main.tf")); + const service = withoutLineComments(readRepoFile("terraform/modules/compute/system_runtime_service.tf")); + + assert.match(rootVars, new RegExp(`variable "${name}"\\s*\\{[^}]*default\\s*=\\s*1\\b`)); + assert.match(moduleVars, new RegExp(`variable "${name}"`)); + assert.match(main, new RegExp(`${name}\\s*=\\s*var\\.${name}\\b`)); + assert.match(service, new RegExp(`desired_count\\s*=\\s*var\\.${name}\\b`)); +}); + test("Terraform ECS services keep Fargate placement and start-before-stop rollout", () => { const cluster = readRepoFile("terraform/modules/compute/cluster.tf"); const locals = readRepoFile("terraform/modules/compute/locals.tf"); diff --git a/tests/unit/workflow-definitions.test.js b/tests/unit/workflow-definitions.test.js new file mode 100644 index 00000000..a3473b37 --- /dev/null +++ b/tests/unit/workflow-definitions.test.js @@ -0,0 +1,63 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { workflowDefinitionsUrl, workflowDefinitionRedisEval } from "../helpers/workflow-definitions.js"; +import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; + +const definitions = await import(workflowDefinitionsUrl); + +test("Workflow definition budgets account replacement bytes and cumulative retired names", () => { + const old = '{"workflowKey":"wf_old","className":"Flow"}'; + const snapshot = { count: definitions.WORKFLOW_DEFINITIONS_MAX_COUNT, bytes: definitions.WORKFLOW_DEFINITIONS_MAX_BYTES, values: [old] }; + assert.equal(definitions.workflowDefinitionUpdatesFit(snapshot, [["flow", old]]), true); + assert.equal(definitions.workflowDefinitionUpdatesFit(snapshot, [["flow", `${old} `]]), false); + assert.equal(definitions.workflowDefinitionUpdatesFit({ ...snapshot, values: [null] }, [["new", old]]), false); +}); + +test("Workflow declaration limits apply after materialized keys and to UTF-8 bytes", () => { + assert.equal(definitions.workflowDeclarationsFit(Array(definitions.WORKFLOW_DEFINITIONS_MAX_COUNT + 1).fill({})), false); + assert.equal(definitions.workflowDeclarationsFit([{ name: "flow", binding: "FLOW", className: "Flow" }]), true); + assert.equal(definitions.workflowDeclarationsFit([{ className: "\u4e2d".repeat(definitions.WORKFLOW_DEFINITIONS_MAX_BYTES / 2) }]), false); +}); + +test("bounded Workflow snapshots expose only requested admission values", async () => { + const redis = createFakeRedis(undefined, { eval: workflowDefinitionRedisEval }); + redis.hashes.set("defs", { selected: "value", unrelated: "not-json" }); + const result = await definitions.readWorkflowDefinitionAdmission(redis, "defs", ["selected", "constructor"]); + assert.equal(result.status, 1); + assert.equal(result.count, 2); + assert.deepEqual(result.values, ["value", null]); + redis.hashes.set("meta", { __meta__: "x".repeat(definitions.WORKFLOW_LIST_META_MAX_BYTES + 1) }); + redis.hashes.set("routes", { worker: "v1" }); + assert.equal((await definitions.readWorkflowDefinitionSnapshot(redis, "defs", "meta", { key: "routes", worker: "worker", version: "v1" })).status, -1); +}); + +test("Workflow snapshots return Lua byte accounting without re-encoding decoded strings", async () => { + const redis = createFakeRedis(undefined, { eval: workflowDefinitionRedisEval }); + const meta = JSON.stringify({ note: "\u4e2d".repeat(2048) }); + const value = JSON.stringify({ className: "\u6587".repeat(2048) }); + redis.hashes.set("routes", { worker: "v1" }); + redis.hashes.set("defs", { flow: value }); + redis.hashes.set("meta", { __meta__: meta }); + const expectedBytes = Buffer.byteLength(meta) + Buffer.byteLength("flow") + Buffer.byteLength(value); + await withMockedProperty(TextEncoder.prototype, "encode", () => { + throw new Error("snapshot must not re-encode strings to count bytes"); + }, async () => { + const snapshot = await definitions.readWorkflowDefinitionSnapshot(redis, "defs", "meta", { key: "routes", worker: "worker", version: "v1" }); + assert.equal(snapshot.status, 1); + assert.equal(snapshot.bytes, expectedBytes); + assert.equal(snapshot.metaRaw, meta); + assert.deepEqual({ ...snapshot.defs }, { flow: value }); + }); +}); + +test("Workflow snapshots report a changed route before reading its old version", async () => { + const redis = createFakeRedis(undefined, { eval: workflowDefinitionRedisEval }); + redis.hashes.set("routes", { worker: "v2" }); + redis.hashes.set("meta", { __meta__: "x".repeat(definitions.WORKFLOW_LIST_META_MAX_BYTES + 1) }); + const snapshot = await definitions.readWorkflowDefinitionSnapshot(redis, "defs", "meta", { key: "routes", worker: "worker", version: "v1" }); + assert.equal(snapshot.status, 2); + assert.equal(snapshot.bytes, 0); + assert.equal(snapshot.metaRaw, null); + assert.deepEqual({ ...snapshot.defs }, {}); +}); diff --git a/tests/unit/workflow-replay-cache.test.js b/tests/unit/workflow-replay-cache.test.js index c663e93d..1bba9f72 100644 --- a/tests/unit/workflow-replay-cache.test.js +++ b/tests/unit/workflow-replay-cache.test.js @@ -53,6 +53,83 @@ async function loadReplayCacheModule() { const CREATED_AT_BASE_MS = 1700000000000; const WORKFLOW_REPLAY_CACHE_STEP_LIMIT = 256; +test("replay working-set admission counts detached state and releases the last controller", async () => { + const mod = await loadReplayCacheModule(); + /** @type {import("../../runtime/dispatch/workflow-replay-cache.js").WorkflowReplayCache[]} */ + const caches = []; + let shared; + for (let index = 0; index < 6; index += 1) { + const cache = mod.acquireWorkflowReplayCache(run(index)); + if (index === 0) shared = mod.acquireWorkflowReplayCache(run(index)); + mod.rememberWorkflowReplayStep(cache, 0, { status: "completed", output: "x".repeat(8 * 1024 * 1024) }); + caches.push(cache); + } + assert.equal(shared, caches[0]); + assert.ok(shared); + const activeBytes = caches.reduce((sum, cache) => sum + cache.bytes, 0); + assert.equal(latestGauge("workflow_replay_active_bytes"), activeBytes); + assert.equal(latestGauge("workflow_replay_working_set_bytes"), activeBytes); + assert.ok(latestGauge("workflow_replay_detached_bytes") > 0); + assert.equal( + latestGauge("workflow_replay_cache_bytes") + latestGauge("workflow_replay_detached_bytes"), + activeBytes, + ); + const lease = mod.createWorkflowReplayReadLease(); + lease.reserve(mod.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES - activeBytes); + assert.throws(() => mod.rememberWorkflowReplayStep(caches[0], 1, { status: "completed", output: "new" }), mod.WorkflowReplayCapacityError); + assert.equal(caches[0].steps.size, 1); + assert.equal(latestGauge("workflow_replay_working_set_bytes"), mod.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES); + mod.releaseWorkflowReplayCache(caches[0]); + assert.equal(latestGauge("workflow_replay_active_bytes"), activeBytes); + const firstBytes = shared.bytes; + mod.releaseWorkflowReplayCache(shared); + assert.equal(latestGauge("workflow_replay_active_bytes"), activeBytes - firstBytes); + assert.equal(shared.bytes, 0); + mod.rememberWorkflowReplayStep(caches[1], 1, { status: "completed", output: "recovered" }); + for (const cache of caches.slice(1)) mod.releaseWorkflowReplayCache(cache); + lease.release(); + lease.release(); + assert.equal(latestGauge("workflow_replay_active_bytes"), 0); + assert.equal(latestGauge("workflow_replay_detached_bytes"), 0); + assert.equal(latestGauge("workflow_replay_read_in_flight_bytes"), 0); + assert.equal(latestGauge("workflow_replay_working_set_bytes"), latestGauge("workflow_replay_cache_bytes")); + assert.equal(latestGauge("workflow_replay_working_set_high_water_bytes"), mod.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES); + assert.ok(metricsState().increments.some((/** @type {any} */ entry) => entry.labels.outcome === "saturated")); +}); + +test("replay read admission evicts unused retained caches and shrinks unknown-length reservations", async () => { + const mod = await loadReplayCacheModule(); + const cache = mod.getWorkflowReplayCache(run(0)); + mod.rememberWorkflowReplayStep(cache, 0, { status: "completed", output: "retained" }); + const lease = mod.createWorkflowReplayReadLease(); + lease.reserve(mod.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES); + assert.equal(cache.released, true); + assert.equal(latestGauge("workflow_replay_cache_bytes"), 0); + lease.reserve(128); + assert.equal(latestGauge("workflow_replay_read_in_flight_bytes"), 128); + lease.release(); + assert.equal(latestGauge("workflow_replay_working_set_bytes"), 0); + assert.equal(latestGauge("workflow_replay_working_set_high_water_bytes"), mod.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES); +}); + +test("replay admission preserves replaced records on refusal and accounts for step eviction", async () => { + const mod = await loadReplayCacheModule(); + const cache = mod.acquireWorkflowReplayCache(run(0)); + for (let ordinal = 0; ordinal < WORKFLOW_REPLAY_CACHE_STEP_LIMIT; ordinal += 1) { + mod.rememberWorkflowReplayStep(cache, ordinal, { status: "completed", output: "old" }); + } + const lease = mod.createWorkflowReplayReadLease(); + lease.reserve(mod.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES - cache.bytes); + assert.throws(() => mod.rememberWorkflowReplayStep(cache, 0, { status: "completed", output: "larger" }), mod.WorkflowReplayCapacityError); + assert.equal(mod.readWorkflowReplayStepOutput(cache.steps.get(0)), "old"); + mod.rememberWorkflowReplayStep(cache, WORKFLOW_REPLAY_CACHE_STEP_LIMIT, { status: "completed", output: "old" }); + assert.equal(cache.steps.has(0), false); + assert.equal(cache.steps.size, WORKFLOW_REPLAY_CACHE_STEP_LIMIT); + assert.equal(latestGauge("workflow_replay_working_set_bytes"), mod.WORKFLOW_REPLAY_WORKING_SET_MAX_BYTES); + mod.releaseWorkflowReplayCache(cache); + lease.release(); +}); + /** @param {number} index */ function run(index) { return {