diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5ae210e4..418dcfc59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,6 +36,9 @@ jobs: - name: Check actual internal fenced workflow with offline effect seams run: pnpm test:chat-fence-workflow + - name: Check dormant pack worker fence with offline Workflow SDK + run: node scripts/pack-fence-workflow-fixture/run.mjs + - name: Check unwired pack scheduler replay with offline effects run: pnpm test:pack-scheduler-workflow diff --git a/app/workflows/packs/foundation/createPackFoundationServices.ts b/app/workflows/packs/foundation/createPackFoundationServices.ts index 55680f12c..17a92c464 100644 --- a/app/workflows/packs/foundation/createPackFoundationServices.ts +++ b/app/workflows/packs/foundation/createPackFoundationServices.ts @@ -1,3 +1,6 @@ +import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes"; +import { assertPackExecutionIdentity } from "@/lib/content/packs/fence/assertPackExecutionIdentity"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; import { preparePackStep } from "../preparePackStep"; import { producePackAssetStep } from "../producePackAssetStep"; import { settleBoundedPackStep } from "../settleBoundedPackStep"; @@ -12,38 +15,66 @@ import type { PackFoundationServices } from "@/lib/content/packs/foundation/pack export function createPackFoundationServices( actor: PackActor, jobId: string, + execution?: PackLaunchFence, ): PackFoundationServices { + if (actor.packProtocol === 2 && !execution) + throw new PackFenceError("Foundation requires original execution admission"); + const attempts = new Map(); const scope = (input: PackActor, id: string) => { if ( id !== jobId || input.accountId !== actor.accountId || - input.organizationId !== actor.organizationId + input.organizationId !== actor.organizationId || + input.packProtocol !== actor.packProtocol ) throw new Error("Foundation service scope mismatch"); }; return { read: (a, id) => { scope(a, id); - return readPackFoundationStep(a, id); + return readPackFoundationStep(a, id).then(saved => { + if (a.packProtocol === 2) { + assertPackExecutionIdentity(saved.job, execution); + for (const asset of saved.assets) + if (!attempts.has(asset.id)) attempts.set(asset.id, asset.attempt); + } + return saved; + }); }, prepare: (a, id) => { scope(a, id); - return preparePackStep(a, id); + return preparePackStep(a, id, execution); }, persist: (a, job, record) => { scope(a, job.id); + if (a.packProtocol === 2) assertPackExecutionIdentity(job, execution); return persistPackFoundationStep(a, job, record); }, produce: (a, job, id) => { scope(a, job.id); + if (a.packProtocol === 2) { + assertPackExecutionIdentity(job, execution); + return producePackAssetStep(a, job, id, false, attempts.get(id)); + } return producePackAssetStep(a, job, id); }, - settle: (a, id, error) => { + settle: (a, id, error, suppliedExecution) => { scope(a, id); - return settleBoundedPackStep(a, id, error); + if ( + a.packProtocol === 2 && + (!suppliedExecution || + suppliedExecution.execution_id !== execution!.execution_id || + suppliedExecution.execution_version !== execution!.execution_version || + suppliedExecution.expected_revision !== execution!.expected_revision) + ) + throw new PackFenceError("Foundation settlement execution mismatch"); + return a.packProtocol === 2 + ? settleBoundedPackStep(a, id, error, suppliedExecution) + : settleBoundedPackStep(a, id, error); }, createPlan: (snapshot, selection) => { scope(actor, snapshot.job.id); + if (actor.packProtocol === 2) assertPackExecutionIdentity(snapshot.job, execution); return planPackFoundationStep(snapshot, selection); }, bindFiles: (plan, asset) => bindPackFoundationFilesStep(actor, jobId, plan, asset), diff --git a/app/workflows/packs/preparePackStep.ts b/app/workflows/packs/preparePackStep.ts index e1607beb0..309bab23b 100644 --- a/app/workflows/packs/preparePackStep.ts +++ b/app/workflows/packs/preparePackStep.ts @@ -1,14 +1,35 @@ +import { assertPackExecutionIdentity } from "@/lib/content/packs/fence/assertPackExecutionIdentity"; +import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; import { getWorkflowMetadata } from "workflow"; import { runPackCommand } from "@/lib/supabase/release_pack_jobs/runPackCommand"; import { preparePackProduction } from "@/lib/content/packs/preparePackProduction"; import type { PackActor } from "@/lib/content/packs/packTypes"; -export async function preparePackStep(actor: PackActor, jobId: string) { +export async function preparePackStep( + actor: PackActor, + jobId: string, + execution?: PackLaunchFence, +) { "use step"; + if (actor.packProtocol === 2 && !execution) + throw new PackFenceError("Missing original launch identity"); + const workflowRunId = getWorkflowMetadata().workflowRunId; await runPackCommand(actor, "run", jobId, { - workflow_run_id: getWorkflowMetadata().workflowRunId, + ...execution, + workflow_run_id: workflowRunId, }); const saved = await runPackCommand(actor, "read", jobId); + if (actor.packProtocol === 2) { + assertPackExecutionIdentity(saved.job, execution, workflowRunId); + if (saved.job.status !== "running") throw new PackFenceError("Pack no longer running"); + } const job = await preparePackProduction(actor, saved.job); - return { job, assetIds: saved.assets.filter(a => a.status === "pending").map(a => a.id) }; + return { + job, + assetIds: saved.assets.filter(a => a.status === "pending").map(a => a.id), + ...(actor.packProtocol === 2 + ? { assetAttempts: Object.fromEntries(saved.assets.map(a => [a.id, a.attempt] as const)) } + : {}), + }; } preparePackStep.maxRetries = 0; diff --git a/app/workflows/packs/producePackAssetStep.ts b/app/workflows/packs/producePackAssetStep.ts index d57168cea..c67c55ebb 100644 --- a/app/workflows/packs/producePackAssetStep.ts +++ b/app/workflows/packs/producePackAssetStep.ts @@ -1,3 +1,9 @@ +import { getWorkflowMetadata } from "workflow"; +import { getPackExecutionFence } from "@/lib/content/packs/fence/getPackExecutionFence"; +import { getPackAssetFence } from "@/lib/content/packs/fence/getPackAssetFence"; +import { getPackClaimRequestId } from "@/lib/content/packs/fence/getPackClaimRequestId"; +import { isPackFenceError } from "@/lib/content/packs/fence/isPackFenceError"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; import { runPackCommand } from "@/lib/supabase/release_pack_jobs/runPackCommand"; import { producePackAsset } from "@/lib/content/packs/producePackAsset"; import { createPackProductionServices } from "@/lib/content/packs/createPackProductionServices"; @@ -7,22 +13,45 @@ export async function producePackAssetStep( job: PackJob, assetId: string, resume = false, + expectedAttempt?: number, ) { "use step"; - const claim = resume - ? await runPackCommand(actor, "read", job.id) - : await runPackCommand(actor, "claim", job.id, { asset_id: assetId }); - const asset = resume ? claim.assets.find(a => a.id === assetId) : claim.asset; - if (!asset || (!resume && !claim.claimed)) return; + const fence = getPackExecutionFence(job); + if (actor.packProtocol === 2 && job.workflow_run_id !== getWorkflowMetadata().workflowRunId) + throw new PackFenceError("Actual workflow does not match original admission"); + if ((actor.packProtocol === 2) !== (job.protocol === 2)) + throw new PackFenceError("Pack protocol mismatch"); + if (actor.packProtocol === 2 && !Number.isInteger(expectedAttempt)) + throw new PackFenceError("Missing original pending attempt"); + const claim = + resume && actor.packProtocol !== 2 + ? await runPackCommand(actor, "read", job.id) + : await runPackCommand(actor, "claim", job.id, { + ...fence, + asset_id: assetId, + ...(actor.packProtocol === 2 + ? { + asset_attempt: expectedAttempt, + request_id: getPackClaimRequestId(job.execution_id!, job.workflow_run_id!, assetId), + } + : {}), + }); + if (actor.packProtocol === 2 && claim.replayed) + throw new PackFenceError("Claim replay is reconciliation, not another production permit"); + const asset = + resume && actor.packProtocol !== 2 ? claim.assets.find(a => a.id === assetId) : claim.asset; + if (!asset || ((actor.packProtocol === 2 || !resume) && !claim.claimed)) return; try { await producePackAsset(job, asset, createPackProductionServices(actor, job, asset)); } catch (error) { + if (isPackFenceError(error)) throw error; const message = error instanceof Error ? error.message : "Asset generation failed"; const uncertain = Object.entries(asset.provider_state).some( ([key, value]) => key.includes("submission") && (value as { state?: string }).state !== "complete", ); await runPackCommand(actor, "finish_asset", job.id, { + ...getPackAssetFence(job, asset), asset_id: assetId, status: uncertain ? "uncertain" : "failed", files: diff --git a/app/workflows/packs/releasePackWorkflow.ts b/app/workflows/packs/releasePackWorkflow.ts index 3c2d3abdc..5f2ad7032 100644 --- a/app/workflows/packs/releasePackWorkflow.ts +++ b/app/workflows/packs/releasePackWorkflow.ts @@ -1,3 +1,5 @@ +import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes"; +import { settleBoundedPackStep } from "./settleBoundedPackStep"; import { preparePackStep } from "./preparePackStep"; import { producePackAssetStep } from "./producePackAssetStep"; import { settlePackStep } from "./settlePackStep"; @@ -7,17 +9,42 @@ export async function releasePackWorkflow( actor: PackActor, jobId: string, resumedAssetId?: string, + execution?: PackLaunchFence, ) { "use workflow"; try { - const prepared = await preparePackStep(actor, jobId); - if (resumedAssetId) await producePackAssetStep(actor, prepared.job, resumedAssetId, true); - else - for (const assetId of prepared.assetIds) - await producePackAssetStep(actor, prepared.job, assetId); - await settlePackStep(actor, jobId); - return { job_id: jobId }; + const prepared = await preparePackStep(actor, jobId, execution); + if (resumedAssetId) { + if (actor.packProtocol === 2) + await producePackAssetStep( + actor, + prepared.job, + resumedAssetId, + true, + prepared.assetAttempts?.[resumedAssetId], + ); + else await producePackAssetStep(actor, prepared.job, resumedAssetId, true); + } else + for (const assetId of prepared.assetIds) { + if (actor.packProtocol === 2) + await producePackAssetStep( + actor, + prepared.job, + assetId, + false, + prepared.assetAttempts?.[assetId], + ); + else await producePackAssetStep(actor, prepared.job, assetId); + } + // Preserve the legacy workflow's existing settlement path. Protocol 2 does not retry an ambiguous settlement. + if (actor.packProtocol !== 2) { + await settlePackStep(actor, jobId); + return { job_id: jobId }; + } } catch (error) { + // Workflow SDK reconstructs terminal step errors as FatalError, losing custom names. + // Any protocol-2 step rejection therefore stops without fallback settlement. + if (actor.packProtocol === 2) throw error; await settlePackStep( actor, jobId, @@ -28,4 +55,6 @@ export async function releasePackWorkflow( error: "Pack stopped; inspect saved status and outputs before retrying", }; } + await settleBoundedPackStep(actor, jobId, undefined, execution); + return { job_id: jobId }; } diff --git a/app/workflows/packs/settleBoundedPackStep.ts b/app/workflows/packs/settleBoundedPackStep.ts index 9c6ad44f9..41ba69fdf 100644 --- a/app/workflows/packs/settleBoundedPackStep.ts +++ b/app/workflows/packs/settleBoundedPackStep.ts @@ -1,8 +1,21 @@ +import { getWorkflowMetadata } from "workflow"; +import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; import { runPackCommand } from "@/lib/supabase/release_pack_jobs/runPackCommand"; import type { PackActor } from "@/lib/content/packs/packTypes"; -/** Unwired scheduler adapter: an ambiguous settlement requires reconciliation, not retry. */ -export async function settleBoundedPackStep(actor: PackActor, jobId: string, error?: string) { +/** Zero-retry adapter used by dormant protocol 2 and the unwired bounded scheduler: an ambiguous settlement requires reconciliation, not retry. */ +export async function settleBoundedPackStep( + actor: PackActor, + jobId: string, + error?: string, + execution?: PackLaunchFence, +) { "use step"; - return runPackCommand(actor, "settle", jobId, { error: error ?? null }); + if (actor.packProtocol === 2 && !execution) + throw new PackFenceError("Missing original settlement identity"); + return runPackCommand(actor, "settle", jobId, { + error: error ?? null, + ...(execution ? { ...execution, workflow_run_id: getWorkflowMetadata().workflowRunId } : {}), + }); } settleBoundedPackStep.maxRetries = 0; diff --git a/lib/content/packs/BOUNDED_SCHEDULER.md b/lib/content/packs/BOUNDED_SCHEDULER.md index e22c4f0c5..ecbe35c1b 100644 --- a/lib/content/packs/BOUNDED_SCHEDULER.md +++ b/lib/content/packs/BOUNDED_SCHEDULER.md @@ -7,9 +7,9 @@ price, approval, reservation, or provider adapter is changed. There is no produc concurrency default or environment switch. The caller supplies a positive safe-integer concurrency bound and the existing -`preparePackStep`, `producePackAssetStep`, and the unwired `settleBoundedPackStep` +`preparePackStep`, `producePackAssetStep`, and `settleBoundedPackStep` services. The settlement adapter uses the same command but sets `maxRetries = 0`; -the existing production `settlePackStep` and its retry behavior are unchanged. +the protocol-1 production `settlePackStep` and its retry behavior are unchanged. The zero-retry adapter is also wired into the dormant protocol-2 production workflow. The controlled fixtures use explicit limits; they do not select a production cap. ## Execution contract @@ -89,3 +89,5 @@ this helper with persisted dependency readiness for an already-quoted artwork as It remains unwired, preserves independent eligibility, and reports explicit owner continuations when prerequisites or reference capacity prevent dependent production. Its source/unit qualification is separate from this helper's compiled replay evidence. + +Protocol-2 exception: after draining in-flight work, any rejected preparation or production step stops without settlement. This conservative rule survives SDK error reconstruction and avoids treating an uncertain write as settlement authority. Successful runs settle once with the original execution fence; a settlement rejection is never retried here. The failure-settlement rules above describe protocol 1. diff --git a/lib/content/packs/__tests__/runBoundedPackProduction.test.ts b/lib/content/packs/__tests__/runBoundedPackProduction.test.ts index a47e5bcbe..dd890230f 100644 --- a/lib/content/packs/__tests__/runBoundedPackProduction.test.ts +++ b/lib/content/packs/__tests__/runBoundedPackProduction.test.ts @@ -410,3 +410,24 @@ it("does not settle while final asset persistence is still in flight", async () expect(assets[0].status).toBe("completed"); expect(settlementActive).toEqual([0]); }); + +it("never forwards an execution fence to legacy settlement", async () => { + const execution = { + expected_revision: 1, + execution_id: "00000000-0000-4000-8000-000000000001", + execution_version: 1, + }; + const settle = vi.fn(async () => undefined); + await runBoundedPackProduction( + { accountId: "owner" }, + "job", + 1, + { + prepare: async () => ({ job: {} as PackJob, assetIds: [] }), + produce: async () => undefined, + settle, + }, + execution, + ); + expect(settle).toHaveBeenCalledWith({ accountId: "owner" }, "job", undefined); +}); diff --git a/lib/content/packs/createPackProductionServices.ts b/lib/content/packs/createPackProductionServices.ts index ad0113928..4df0d1755 100644 --- a/lib/content/packs/createPackProductionServices.ts +++ b/lib/content/packs/createPackProductionServices.ts @@ -1,3 +1,7 @@ +import { createHash } from "node:crypto"; +import { getPackAssetFence } from "./fence/getPackAssetFence"; +import { PackFenceError } from "./fence/PackFenceError"; +import type { PackProviderReceipt } from "./fence/packFenceTypes"; import { packExecutionSettings, PACK_CAPTION_MODEL } from "@/lib/const"; import { assertPackQuoteConfiguration } from "./assertPackQuoteConfiguration"; import { generateText } from "ai"; @@ -20,9 +24,47 @@ export function createPackProductionServices( asset: PackAsset, ): PackProductionServices { const state = asset.provider_state; + const fence = getPackAssetFence(job, asset); + if ((actor.packProtocol === 2) !== (job.protocol === 2)) + throw new PackFenceError("Pack protocol mismatch"); + const observe = + actor.packProtocol === 2 + ? async (receipt: PackProviderReceipt) => { + if (!receipt.operation_id || !receipt.request_id) + throw new PackFenceError("Missing durable provider operation"); + const { appendPackProviderObservation } = await import( + "@/lib/supabase/release_pack_jobs/appendPackProviderObservation" + ); + await appendPackProviderObservation( + actor, + receipt.operation_id, + { execution_id: fence.execution_id, claim_id: fence.claim_id }, + { + kind: receipt.state === "complete" ? "completed" : "accepted", + request_id: receipt.request_id, + ...(receipt.state === "complete" && receipt.data !== undefined + ? { + output_sha256: createHash("sha256") + .update(JSON.stringify(receipt.data)) + .digest("hex"), + } + : {}), + }, + ); + } + : undefined; const phase = async (name: string, value: unknown) => { - await runPackCommand(actor, "phase", job.id, { asset_id: asset.id, phase: name, value }); + const saved = await runPackCommand(actor, "phase", job.id, { + ...fence, + asset_id: asset.id, + phase: name, + value, + ...(actor.packProtocol === 2 && job.shared_foundation_binding + ? { shared_foundation_binding: job.shared_foundation_binding } + : {}), + }); state[name] = value; + return saved; }; return { phase, @@ -42,6 +84,7 @@ export function createPackProductionServices( key: `image_submission_${index}`, state, save: phase, + observe, }); const url = (data.images as { url?: string }[] | undefined)?.[0]?.url; if (!url) throw new Error("House image model returned no image"); @@ -69,6 +112,7 @@ export function createPackProductionServices( key: "motion_submission", state, save: phase, + observe, }); const url = (data.video as { url?: string } | undefined)?.url; if (!url) throw new Error("House video model returned no video"); @@ -97,6 +141,6 @@ export function createPackProductionServices( return inspectPackMedia(input); }, finish: result => - runPackCommand(actor, "finish_asset", job.id, { asset_id: asset.id, ...result }), + runPackCommand(actor, "finish_asset", job.id, { ...result, ...fence, asset_id: asset.id }), }; } diff --git a/lib/content/packs/fence/PackFenceError.ts b/lib/content/packs/fence/PackFenceError.ts new file mode 100644 index 000000000..3bc4aeebb --- /dev/null +++ b/lib/content/packs/fence/PackFenceError.ts @@ -0,0 +1,7 @@ +/** An uncertain/refused durable write must not trigger fallback finish, settle or paid retry. */ +export class PackFenceError extends Error { + constructor(message = "Protocol-2 persistence requires reconciliation") { + super(message); + this.name = "PackFenceError"; + } +} diff --git a/lib/content/packs/fence/README.md b/lib/content/packs/fence/README.md new file mode 100644 index 000000000..baf98d468 --- /dev/null +++ b/lib/content/packs/fence/README.md @@ -0,0 +1,44 @@ +# Dormant protocol-2 Pack execution fence + +This source is opt-in through the **internal fifth argument** to `processPackOperation`. Existing REST/MCP callers do not provide it, no public schema accepts it, and no environment flag enables it. Public and legacy launches remain protocol 1. This change does not provide a stale-worker guarantee for legacy executions. + +The paired Database branch `codex/pack-worker-fence-2269` contains inactive SQL under `proposals/pack-worker-fence-2269/`; nothing is placed in a migration/deployment runner. Root owns paired acceptance, merge and later rollout. Do not enable protocol 2 against an unqualified hosted schema. + +## Original identity, not current-row adoption + +An admitted transition carries a stable request UUID, source execution UUID/version, and (for resume) source claim and paid attempt. PostgreSQL issues the next execution UUID/version only once; exact transition replay never dispatches or holds again. The launch fence is passed through Workflow arguments. Preparation validates it after its read; production also compares the actual Workflow metadata before claiming. Asset writes use the issued claim and original paid attempt. Context updates compare context version. No helper fills missing identity from a newer job. + +Claim replay is a reconciliation result, not a second production permit. Uncertain resume claims the same paid attempt under a new execution and preserves provider IDs and billing. Duplicate intent registration is likewise not permission to submit again. + +## Receipts and failure behavior + +The actual production adapter registers frozen provider intent in the fenced phase transaction, submits only after a fresh acknowledgement, and independently appends an allowlisted accepted/completed observation before saving the active phase. Full results remain in the existing phase path; only request ID and optional output hash enter the bounded journal. If append or active save is refused/uncertain, stop; do not mark a catch failure, settle a newer execution or resubmit the provider request. + +The Workflow SDK reconstructs step errors. Protocol 2 therefore stops on any rejected step, not only `instanceof PackFenceError`. Its settlement uses the existing zero-retry bounded step outside the catch path. Foundation/bounded adapters propagate original identity and attempts; retained foundation dependents pass their plan/asset binding for SQL model validation. The bounded scheduler, foundation orchestration and lyric renderer remain unwired. The zero-retry settlement adapter is wired into the dormant protocol-2 release workflow. + +## Local checks + +```sh +pnpm exec vitest run lib/content/packs app/workflows/packs +pnpm typecheck:release-packs +pnpm lint +node scripts/pack-fence-workflow-fixture/run.mjs +pnpm test:pack-scheduler-workflow +pnpm test:pack-admission-workflow +pnpm test:pack-foundation-workflow +``` + +`scripts/pack-fence-workflow-fixture/run.mjs` compiles the actual production workflow and steps with the installed Workflow SDK, replaces only persistence/provider leaves with synthetic effects, denies networking, and verifies rejection, one settlement attempt after lost acknowledgement, and terminal replay. `PACK_FENCE_SDK_OUTPUT=/absolute/path.json` writes source hashes and events. + +For actual paired SQL integration, put PostgreSQL 17 binaries on PATH and supply the Database checkout explicitly: + +```sh +python3 scripts/pack-fence-postgres/run.py --database /absolute/path/to/database-checkout --print-binding +PYTHONDONTWRITEBYTECODE=1 python3 scripts/pack-fence-postgres/run.py --database /absolute/path/to/database-checkout --expected-binding +``` + +That runner creates and tears down an owned temporary Unix-socket PostgreSQL instance, applies the existing synthetic fixture and inactive proposal, and runs `packFencePostgres.test.ts`. Only Supabase RPC transport is adapted to that local database; production claim/intent/receipt/phase/completion helpers are real. Provider, caption, composition and inspection effects are synthetic. The six integration tests are skipped in ordinary Vitest runs and must be qualified through this explicit runner. They cover concurrent foundation CAS with one winner and acknowledged version advancement, charging/completion and claim replay, actual Workflow mismatch, foundation-dependent image-edit and rejection of unretained bindings, and a committed acceptance surviving refused phase without catch finish or charge. + +These checks do not establish hosted grants, deployed behavior, live provider execution, cancellation of in-flight I/O, automatic reconciliation, cache/QA freshness, legacy drain or customer outcomes. Unknown intent is not safe to resubmit. No new wallet, rates, paid runtime or public activation is included. + +The paired runner binds Database HEAD and fixture/proposal bytes (including explicitly reviewed uncommitted candidates) before import and verifies they remain unchanged after testing. It strips credentials from child environments and fails on unconfirmed cleanup. Protocol 2 intentionally treats every RPC refusal as a stop/reconciliation signal; public 402/404 mapping must be decided before activation. Deterministic claim request IDs are opaque stable identities, not claims of random UUID generation; replacing their algorithm requires a replay-compatibility decision before activation. diff --git a/lib/content/packs/fence/__tests__/packFencePostgres.test.ts b/lib/content/packs/fence/__tests__/packFencePostgres.test.ts new file mode 100644 index 000000000..7da856796 --- /dev/null +++ b/lib/content/packs/fence/__tests__/packFencePostgres.test.ts @@ -0,0 +1,343 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { savePackFoundationRecord } from "../../foundation/savePackFoundationRecord"; +import { createPackFoundationPlan } from "../../foundation/createPackFoundationPlan"; +import { foundationFixture } from "../../foundation/__tests__/foundationFixture"; +import { randomUUID } from "node:crypto"; +import { userInfo } from "node:os"; +import { Client } from "pg"; +import { runPackCommand } from "@/lib/supabase/release_pack_jobs/runPackCommand"; +import { producePackAssetStep } from "@/app/workflows/packs/producePackAssetStep"; +import { buildReleasePackQuote } from "../../buildReleasePackQuote"; +import { getPackExecutionFence } from "../getPackExecutionFence"; +import { readPackProviderObservations } from "@/lib/supabase/release_pack_jobs/readPackProviderObservations"; +import type { PackActor } from "../../packTypes"; +const hooks = vi.hoisted(() => ({ + beforeContext: undefined as undefined | (() => Promise), + beforePhase: undefined as undefined | (() => Promise), + submit: vi.fn(), + runId: "", + trace: [] as string[], +})); +vi.mock("@/lib/supabase/serverClient", () => ({ + default: { + rpc: async (name: string, args: Record) => { + if ( + ![ + "release_pack_command_v2", + "release_pack_command", + "append_pack_provider_observation", + "read_pack_provider_observations", + ].includes(name) + ) + throw new Error("Unexpected local RPC"); + hooks.trace.push(name + (args.p_action ? ":" + args.p_action : "")); + if ( + name === "release_pack_command_v2" && + args.p_action === "phase" && + (args.p_input as { value?: { state?: string } }).value?.state === "accepted" && + hooks.beforePhase + ) { + const act = hooks.beforePhase; + hooks.beforePhase = undefined; + await act(); + } + if (name === "release_pack_command_v2" && args.p_action === "context" && hooks.beforeContext) + await hooks.beforeContext(); + const client = await localClient(); + try { + return { + data: ( + await client.query( + `SELECT public.${name}(${Object.keys(args) + .map((_, i) => `$${i + 1}`) + .join(",")}) AS result`, + Object.values(args), + ) + ).rows[0].result, + error: null, + }; + } catch (error) { + return { data: null, error }; + } finally { + await client.end(); + } + }, + }, +})); +vi.mock("workflow", () => ({ getWorkflowMetadata: () => ({ workflowRunId: hooks.runId }) })); +vi.mock("@/lib/fal/server", () => ({ + default: { + queue: { + submit: hooks.submit, + status: async () => ({ status: "COMPLETED" }), + result: async () => ({ data: { images: [{ url: "https://example.invalid/original" }] } }), + }, + }, +})); +vi.mock("ai", () => ({ generateText: async () => ({ text: "Synthetic caption" }) })); +vi.mock("@/lib/content/packs/composePackMedia", () => ({ + composePackMedia: async () => ({ + files: [{ storage_key: "synthetic/output", content_type: "image/png", bytes: 1 }], + probe: {}, + }), +})); +vi.mock("@/lib/content/packs/inspectPackMedia", () => ({ + inspectPackMedia: async () => ({ passed: true, reason: "Synthetic offline inspection" }), +})); +async function localClient(admin = false) { + const socket = process.env.PACK_FENCE_TEST_SOCKET; + if (!socket || !socket.includes("/recoup-pack-")) + throw new Error("Owned local PG socket required"); + const c = new Client({ + host: socket, + port: Number(process.env.PACK_FENCE_TEST_PORT), + database: "postgres", + user: userInfo().username, + }); + await c.connect(); + if (!admin) await c.query("SET ROLE service_role"); + return c; +} +async function seed(foundation = false) { + const actor: PackActor = { accountId: randomUUID(), packProtocol: 2 }, + jobId = randomUUID(); + const c = await localClient(true); + try { + await c.query("INSERT INTO accounts(id) VALUES($1);", [actor.accountId]); + await c.query("INSERT INTO credits_usage(account_id,remaining_credits) VALUES($1,10000000)", [ + actor.accountId, + ]); + } finally { + await c.end(); + } + const brief = { + artist_name: "Fixture", + song_title: "Fixture", + primary_action: "listen", + allow_creative_defaults: false, + reference_image_urls: [], + platforms: ["instagram" as const], + evidence: [], + constraints: [], + audio_url: "https://example.invalid/audio", + }; + const deliverables = { + videos: 0, + artwork: 1, + quote_cards: foundation ? 1 : 0, + carousels: 0, + visualizers: 0, + video_seconds: 8, + }; + await runPackCommand(actor, "prepare", jobId, { + idempotency_key: jobId, + fingerprint: "a".repeat(64), + brief, + deliverables, + status: "ready_to_quote", + }); + const quote = { + ...buildReleasePackQuote(brief, deliverables), + id: randomUUID(), + revision: 1, + expires_at: "2099-01-01T00:00:00Z", + }; + await runPackCommand(actor, "estimate", jobId, { + expected_revision: 1, + quote, + assets: quote.assets, + }); + const funded = await runPackCommand(actor, "approve", jobId, { + request_id: randomUUID(), + expected_revision: 1, + source_execution_id: null, + source_execution_version: 0, + quote_id: quote.id, + approved_max_usd: quote.maximum_usd, + }); + const bound = await runPackCommand(actor, "run", jobId, { + expected_revision: 1, + execution_id: funded.job.execution_id, + execution_version: funded.job.execution_version, + workflow_run_id: randomUUID(), + }); + hooks.runId = bound.job.workflow_run_id!; + return { actor, job: bound.job, asset: bound.assets[0], assets: bound.assets }; +} +describe.skipIf(!process.env.PACK_FENCE_TEST_SOCKET)( + "paired source + actual proposed PostgreSQL", + () => { + beforeEach(() => { + hooks.beforeContext = undefined; + hooks.beforePhase = undefined; + hooks.trace = []; + hooks.submit.mockReset().mockResolvedValue({ request_id: "original-request" }); + }); + it("refuses competing foundation writes at the actual SQL CAS and retains acknowledged versions", async () => { + const { actor, job } = await seed(true); + const initialized = await runPackCommand(actor, "context", job.id, { + ...getPackExecutionFence(job), + context_version: job.context_version, + value: foundationFixture().job.production_context, + }); + const record = { + plan: createPackFoundationPlan(initialized, { dependentAssetIds: [] }), + state: "pending" as const, + files: [], + }; + let arrivals = 0; + let release!: () => void; + let bothArrived!: () => void; + const barrier = new Promise(resolve => { + release = resolve; + }); + const entered = new Promise(resolve => { + bothArrived = resolve; + }); + hooks.beforeContext = async () => { + if (++arrivals === 2) bothArrived(); + await barrier; + }; + const attempts = Promise.allSettled([ + savePackFoundationRecord(actor, initialized.job, record), + savePackFoundationRecord(actor, initialized.job, record), + ]); + await entered; + release(); + const outcomes = await attempts; + hooks.beforeContext = undefined; + expect(outcomes.filter(result => result.status === "fulfilled")).toHaveLength(1); + expect(outcomes.filter(result => result.status === "rejected")).toHaveLength(1); + const current = await runPackCommand(actor, "read", job.id); + expect(current.job.context_version).toBe(initialized.job.context_version! + 1); + await expect(savePackFoundationRecord(actor, initialized.job, record)).rejects.toThrow( + "context version changed", + ); + const next = await savePackFoundationRecord(actor, current.job, record); + expect(next.context_version).toBe(current.job.context_version! + 1); + expect(hooks.submit).not.toHaveBeenCalled(); + }); + it("uses real claim/intent/journal/charge/completion and never resubmits a replayed claim", async () => { + const { actor, job, asset } = await seed(); + await producePackAssetStep(actor, job, asset.id, false, 0); + const saved = await runPackCommand(actor, "read", job.id); + expect(saved.assets[0].status).toBe("completed"); + expect(saved.job.consumed_credits).toBe(asset.credits); + const page = await readPackProviderObservations(actor, job.id); + expect(page.operations).toHaveLength(1); + expect((page.operations[0].observations as { kind: string }[]).map(x => x.kind)).toEqual([ + "accepted", + "completed", + ]); + await expect(producePackAssetStep(actor, job, asset.id, false, 0)).rejects.toThrow( + "reconciliation", + ); + expect(hooks.submit).toHaveBeenCalledTimes(1); + await runPackCommand(actor, "settle", job.id, getPackExecutionFence(job)); + }); + it("rejects a different actual workflow before claim or provider effects", async () => { + const { actor, job, asset } = await seed(); + hooks.runId = "different-run"; + hooks.trace = []; + await expect(producePackAssetStep(actor, job, asset.id, false, 0)).rejects.toThrow( + "workflow", + ); + expect(hooks.trace).toEqual([]); + expect(hooks.submit).not.toHaveBeenCalled(); + }); + it("accepts dependent image-edit only with the retained ready foundation binding", async () => { + const { actor, job, assets } = await seed(true); + const artwork = assets.find(a => a.kind === "artwork")!, + dependent = assets.find(a => a.kind === "quote_card")!; + await producePackAssetStep(actor, job, artwork.id, false, 0); + const binding = { plan_sha256: "b".repeat(64), asset_sha256: "c".repeat(64) }; + const context = { + shared_foundation: { + state: "ready", + plan: { + digest: binding.plan_sha256, + job_id: job.id, + owner_id: job.owner_id, + revision: job.revision, + quote_id: job.quote!.id, + asset_id: artwork.id, + dependent_asset_ids: [dependent.id], + }, + asset_sha256: binding.asset_sha256, + dependency_bindings: [{ asset_id: dependent.id, attempt: 1 }], + files: [{ storage_key: "synthetic/output", sha256: "d".repeat(64) }], + }, + }; + await runPackCommand(actor, "context", job.id, { + ...getPackExecutionFence(job), + context_version: job.context_version, + value: context, + }); + const dependentJob = { + ...job, + brief: { + ...job.brief, + reference_image_urls: ["https://example.invalid/retained-foundation"], + }, + shared_foundation_binding: binding, + }; + await producePackAssetStep(actor, dependentJob, dependent.id, false, 0); + const saved = await runPackCommand(actor, "read", job.id); + expect(saved.assets.every(a => a.status === "completed")).toBe(true); + expect(hooks.submit).toHaveBeenCalledTimes(2); + const page = await readPackProviderObservations(actor, job.id); + expect(page.operations.map(o => o.model).sort()).toEqual( + [ + job.quote!.execution_config.models.image, + job.quote!.execution_config.models.image_edit, + ].sort(), + ); + }); + it("refuses an unretained foundation binding before dependent submission", async () => { + const { actor, job, assets } = await seed(true); + const artwork = assets.find(a => a.kind === "artwork")!; + const dependent = assets.find(a => a.kind === "quote_card")!; + await producePackAssetStep(actor, job, artwork.id, false, 0); + const before = await runPackCommand(actor, "read", job.id); + await expect( + producePackAssetStep( + actor, + { + ...job, + brief: { ...job.brief, reference_image_urls: ["https://example.invalid/unretained"] }, + shared_foundation_binding: { + plan_sha256: "b".repeat(64), + asset_sha256: "c".repeat(64), + }, + }, + dependent.id, + false, + 0, + ), + ).rejects.toThrow("reconciliation"); + const after = await runPackCommand(actor, "read", job.id); + expect(after.job.consumed_credits).toBe(before.job.consumed_credits); + expect(after.assets.find(a => a.id === dependent.id)?.provider_state).toEqual({}); + expect((await readPackProviderObservations(actor, job.id)).operations).toHaveLength(1); + expect(hooks.submit).toHaveBeenCalledTimes(1); + }); + it("retains acceptance before actual stale phase refusal without catch finish or charge", async () => { + const { actor, job, asset } = await seed(); + hooks.beforePhase = async () => { + await runPackCommand(actor, "settle", job.id, getPackExecutionFence(job)); + }; + await expect(producePackAssetStep(actor, job, asset.id, false, 0)).rejects.toThrow( + "reconciliation", + ); + const saved = await runPackCommand(actor, "read", job.id); + expect(saved.job.consumed_credits).toBe(0); + expect(saved.assets[0].status).toBe("uncertain"); + const page = await readPackProviderObservations(actor, job.id); + expect((page.operations[0].observations as { kind: string }[]).map(x => x.kind)).toEqual([ + "accepted", + ]); + expect(hooks.trace.filter(x => x.endsWith(":finish_asset"))).toHaveLength(0); + expect(hooks.submit).toHaveBeenCalledTimes(1); + }); + }, +); diff --git a/lib/content/packs/fence/__tests__/packPreparationFence.test.ts b/lib/content/packs/fence/__tests__/packPreparationFence.test.ts new file mode 100644 index 000000000..b5b0b4ef5 --- /dev/null +++ b/lib/content/packs/fence/__tests__/packPreparationFence.test.ts @@ -0,0 +1,39 @@ +import { expect, it, vi } from "vitest"; +import { preparePackStep } from "@/app/workflows/packs/preparePackStep"; +const mocks = vi.hoisted(() => ({ command: vi.fn(), prepare: vi.fn() })); +vi.mock("workflow", () => ({ getWorkflowMetadata: () => ({ workflowRunId: "original-run" }) })); +vi.mock("@/lib/supabase/release_pack_jobs/runPackCommand", () => ({ + runPackCommand: mocks.command, +})); +vi.mock("@/lib/content/packs/preparePackProduction", () => ({ + preparePackProduction: mocks.prepare, +})); +it("refuses rotation between run acknowledgement and fresh read before preparation", async () => { + const original = { + expected_revision: 1, + execution_id: "00000000-0000-4000-8000-000000000001", + execution_version: 1, + }; + mocks.command.mockResolvedValueOnce({}).mockResolvedValueOnce({ + job: { + protocol: 2, + revision: 1, + execution_id: "00000000-0000-4000-8000-000000000002", + execution_version: 2, + workflow_run_id: "new-run", + status: "running", + }, + assets: [], + }); + await expect( + preparePackStep({ accountId: "owner", packProtocol: 2 }, "job", original), + ).rejects.toThrow("Pack execution changed"); + expect(mocks.prepare).not.toHaveBeenCalled(); + expect(mocks.command).toHaveBeenNthCalledWith( + 1, + { accountId: "owner", packProtocol: 2 }, + "run", + "job", + { ...original, workflow_run_id: "original-run" }, + ); +}); diff --git a/lib/content/packs/fence/__tests__/packWorkerFence.test.ts b/lib/content/packs/fence/__tests__/packWorkerFence.test.ts new file mode 100644 index 000000000..727ad0c5a --- /dev/null +++ b/lib/content/packs/fence/__tests__/packWorkerFence.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it, vi } from "vitest"; +import { getPackExecutionFence } from "../getPackExecutionFence"; +import { getPackAssetFence } from "../getPackAssetFence"; +import { runPackFalRequest } from "../../runPackFalRequest"; +const provider = vi.hoisted(() => ({ submit: vi.fn(), status: vi.fn(), result: vi.fn() })); +vi.mock("@/lib/fal/server", () => ({ default: { queue: provider } })); +const id = "00000000-0000-4000-8000-000000000001"; +const claimId = "00000000-0000-4000-8000-000000000002"; +const job = { + protocol: 2, + revision: 1, + execution_id: id, + execution_version: 2, + workflow_run_id: "run-original", +}; +describe("dormant Pack worker fence", () => { + it("retains original execution and claim; never fills missing identity", () => { + expect(getPackExecutionFence(job)).toEqual({ + expected_revision: 1, + execution_id: id, + execution_version: 2, + workflow_run_id: "run-original", + }); + expect(getPackAssetFence(job, { attempt: 3, claim_id: claimId })).toMatchObject({ + asset_attempt: 3, + claim_id: claimId, + }); + expect(() => getPackExecutionFence({ ...job, execution_id: undefined })).toThrow(); + expect(() => getPackAssetFence(job, { attempt: 3 })).toThrow(); + expect(getPackExecutionFence({ revision: 1 })).toEqual({}); + }); + it("commits accepted observation before attempting phase; never polls on stale refusal", async () => { + const order: string[] = []; + provider.submit.mockResolvedValue({ request_id: "accepted-original" }); + const save = vi.fn(async (_key, receipt) => { + order.push(`phase:${receipt.state}`); + if (receipt.state === "accepted") throw new Error("stale"); + return { operation_id: id, operation_created: true }; + }); + let acknowledge!: () => void; + let entered!: () => void; + const observationEntered = new Promise(resolve => { + entered = resolve; + }); + const acknowledgement = new Promise(resolve => { + acknowledge = resolve; + }); + const pending = runPackFalRequest({ + model: "fixture", + input: {}, + chargeCredits: 1, + key: "image_submission_0", + state: {}, + save, + observe: async receipt => { + order.push(`observation:${receipt.state}`); + entered(); + await acknowledgement; + }, + }); + const rejected = expect(pending).rejects.toThrow("stale"); + await observationEntered; + expect(order).toEqual(["phase:submitting", "observation:accepted"]); + expect(save).toHaveBeenCalledTimes(1); + acknowledge(); + await rejected; + expect(order).toEqual(["phase:submitting", "observation:accepted", "phase:accepted"]); + expect(provider.status).not.toHaveBeenCalled(); + }); + it("never submits again on a duplicate or lost-ack intent", async () => { + provider.submit.mockClear(); + await expect( + runPackFalRequest({ + model: "fixture", + input: {}, + chargeCredits: 1, + key: "image_submission_0", + state: {}, + save: async () => ({ operation_id: id, operation_created: false }), + observe: async () => {}, + }), + ).rejects.toThrow("without resubmission"); + expect(provider.submit).not.toHaveBeenCalled(); + }); + it("does not submit if durable intent registration fails", async () => { + provider.submit.mockClear(); + await expect( + runPackFalRequest({ + model: "fixture", + input: {}, + chargeCredits: 1, + key: "image_submission_0", + state: {}, + save: async () => { + throw new Error("registration failed"); + }, + observe: async () => {}, + }), + ).rejects.toThrow("registration failed"); + expect(provider.submit).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/content/packs/fence/assertPackExecutionIdentity.ts b/lib/content/packs/fence/assertPackExecutionIdentity.ts new file mode 100644 index 000000000..5d97eaa50 --- /dev/null +++ b/lib/content/packs/fence/assertPackExecutionIdentity.ts @@ -0,0 +1,19 @@ +import { getPackLaunchFence } from "./getPackLaunchFence"; +import type { PackLaunchFence } from "./packFenceTypes"; +import { PackFenceError } from "./PackFenceError"; +export function assertPackExecutionIdentity( + job: { [key: string]: unknown; protocol?: number; workflow_run_id?: string | null }, + original: PackLaunchFence | undefined, + workflowId?: string, +) { + if (job.protocol !== 2) throw new PackFenceError("Execution fence requires protocol 2"); + if (!original) throw new PackFenceError("Missing original execution identity"); + const actual = getPackLaunchFence(job); + if ( + actual.execution_id !== original.execution_id || + actual.execution_version !== original.execution_version || + actual.expected_revision !== original.expected_revision || + (workflowId !== undefined && job.workflow_run_id !== workflowId) + ) + throw new PackFenceError("Pack execution changed before effects"); +} diff --git a/lib/content/packs/fence/getPackAssetFence.ts b/lib/content/packs/fence/getPackAssetFence.ts new file mode 100644 index 000000000..fcb64cc9b --- /dev/null +++ b/lib/content/packs/fence/getPackAssetFence.ts @@ -0,0 +1,18 @@ +import type { PackAssetFence } from "./packFenceTypes"; +import { z } from "zod"; +import { getPackExecutionFence } from "./getPackExecutionFence"; +import { PackFenceError } from "./PackFenceError"; +const claim = z.object({ attempt: z.number().int().positive(), claim_id: z.uuid() }); +export function getPackAssetFence( + job: Parameters[0], + asset: { attempt?: number; claim_id?: string }, +): Partial { + if (job.protocol !== 2) return {}; + const parsed = claim.safeParse(asset); + if (!parsed.success) throw new PackFenceError("Missing original Pack asset claim"); + return { + ...getPackExecutionFence(job), + asset_attempt: parsed.data.attempt, + claim_id: parsed.data.claim_id, + }; +} diff --git a/lib/content/packs/fence/getPackClaimRequestId.ts b/lib/content/packs/fence/getPackClaimRequestId.ts new file mode 100644 index 000000000..11243fdaa --- /dev/null +++ b/lib/content/packs/fence/getPackClaimRequestId.ts @@ -0,0 +1,8 @@ +import { createHash } from "node:crypto"; +/** Deterministic per workflow/asset acquisition; replay never invents a fresh claim. */ +export function getPackClaimRequestId(executionId: string, workflowId: string, assetId: string) { + const hex = createHash("sha256") + .update(JSON.stringify([executionId, workflowId, assetId])) + .digest("hex"); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-8${hex.slice(17, 20)}-${hex.slice(20, 32)}`; +} diff --git a/lib/content/packs/fence/getPackExecutionFence.ts b/lib/content/packs/fence/getPackExecutionFence.ts new file mode 100644 index 000000000..415d4e035 --- /dev/null +++ b/lib/content/packs/fence/getPackExecutionFence.ts @@ -0,0 +1,13 @@ +import { getPackLaunchFence } from "./getPackLaunchFence"; +import { PackFenceError } from "./PackFenceError"; +export function getPackExecutionFence(job: { + protocol?: number; + revision?: number; + execution_id?: string; + execution_version?: number; + workflow_run_id?: string | null; +}) { + if (job.protocol !== 2) return {}; + if (!job.workflow_run_id) throw new PackFenceError("Missing original Pack workflow identity"); + return { ...getPackLaunchFence(job), workflow_run_id: job.workflow_run_id }; +} diff --git a/lib/content/packs/fence/getPackLaunchFence.ts b/lib/content/packs/fence/getPackLaunchFence.ts new file mode 100644 index 000000000..89c622509 --- /dev/null +++ b/lib/content/packs/fence/getPackLaunchFence.ts @@ -0,0 +1,16 @@ +import { z } from "zod"; +import { PackFenceError } from "./PackFenceError"; +const identity = z.object({ + revision: z.number().int().positive(), + execution_id: z.uuid(), + execution_version: z.number().int().positive(), +}); +export function getPackLaunchFence(job: unknown) { + const parsed = identity.safeParse(job); + if (!parsed.success) throw new PackFenceError("Missing original Pack execution identity"); + return { + expected_revision: parsed.data.revision, + execution_id: parsed.data.execution_id, + execution_version: parsed.data.execution_version, + }; +} diff --git a/lib/content/packs/fence/getPackTransitionInput.ts b/lib/content/packs/fence/getPackTransitionInput.ts new file mode 100644 index 000000000..a9b0347b1 --- /dev/null +++ b/lib/content/packs/fence/getPackTransitionInput.ts @@ -0,0 +1,18 @@ +import { z } from "zod"; +import type { PackProtocol2Admission } from "./packFenceTypes"; +import { PackFenceError } from "./PackFenceError"; +const transition = z.strictObject({ + request_id: z.uuid(), + source_execution_id: z.uuid().nullable(), + source_execution_version: z.number().int().nonnegative(), + source_claim_id: z.uuid().optional(), + asset_attempt: z.number().int().nonnegative().optional(), +}); +/** Original caller admission, never repaired by reading current execution state. */ +export function getPackTransitionInput(admission: PackProtocol2Admission | undefined) { + if (!admission) return {}; + const result = transition.safeParse(admission.transition); + if (!result.success) + throw new PackFenceError("Original protocol-2 transition admission required"); + return result.data; +} diff --git a/lib/content/packs/fence/isPackFenceError.ts b/lib/content/packs/fence/isPackFenceError.ts new file mode 100644 index 000000000..e12c87914 --- /dev/null +++ b/lib/content/packs/fence/isPackFenceError.ts @@ -0,0 +1,3 @@ +export function isPackFenceError(error: unknown): boolean { + return error instanceof Error && error.name === "PackFenceError"; +} diff --git a/lib/content/packs/fence/packFenceTypes.ts b/lib/content/packs/fence/packFenceTypes.ts new file mode 100644 index 000000000..fd1c81b4f --- /dev/null +++ b/lib/content/packs/fence/packFenceTypes.ts @@ -0,0 +1,27 @@ +/** Internal only: never accepted from public Pack brief or route input. */ +export type PackLaunchFence = { + expected_revision: number; + execution_id: string; + execution_version: number; +}; +export type PackExecutionFence = PackLaunchFence & { workflow_run_id: string }; +export type PackAssetFence = PackExecutionFence & { asset_attempt: number; claim_id: string }; +export type PackProtocol2Admission = { + protocol: 2; + transition?: { + request_id: string; + source_execution_id: string | null; + source_execution_version: number; + source_claim_id?: string; + asset_attempt?: number; + }; +}; +export type PackProviderReceipt = { + state: string; + operation_id?: string; + request_id?: string; + data?: unknown; + model?: string; + input_sha256?: string; + charge_credits?: number; +}; diff --git a/lib/content/packs/foundation/__tests__/packFoundationBatch.test.ts b/lib/content/packs/foundation/__tests__/packFoundationBatch.test.ts index 52bb2e997..b5c8452f2 100644 --- a/lib/content/packs/foundation/__tests__/packFoundationBatch.test.ts +++ b/lib/content/packs/foundation/__tests__/packFoundationBatch.test.ts @@ -1,3 +1,4 @@ +import type { PackJob } from "../../packTypes"; import { createPackFoundationPlan } from "../createPackFoundationPlan"; import { bindPackFoundationFiles } from "../bindPackFoundationFiles"; import { foundationImageBytes } from "./foundationImageBytes"; @@ -25,12 +26,18 @@ function setup(artwork = 1) { assetIds: snapshot.assets.filter(a => a.status === "pending").map(a => a.id), })), read: vi.fn(async () => structuredClone(snapshot)), - persist: vi.fn(async (_actor: unknown, _job: unknown, record: PackFoundationRecord) => { - snapshot.job.production_context = { - ...(snapshot.job.production_context as object), - shared_foundation: JSON.parse(JSON.stringify(record)), - }; - }), + persist: vi.fn( + async ( + _actor: unknown, + _job: unknown, + record: PackFoundationRecord, + ): Promise => { + snapshot.job.production_context = { + ...(snapshot.job.production_context as object), + shared_foundation: JSON.parse(JSON.stringify(record)), + }; + }, + ), produce: vi.fn(async (_actor: unknown, job: typeof snapshot.job, id: string) => { produced.push({ id, @@ -382,3 +389,152 @@ it("runs plan hashing and retained-byte binding only through the supplied durabl f.services.persist.mock.invocationCallOrder.at(-1)!, ); }); + +it("propagates acknowledged preparation and persistence versions and the settlement fence", async () => { + const f = setup(); + const actor = { ...f.actor, packProtocol: 2 as const }; + const execution = { + expected_revision: 1, + execution_id: "00000000-0000-4000-8000-000000000001", + execution_version: 1, + }; + Object.assign(f.snapshot.job, { + protocol: 2, + ...execution, + revision: 1, + workflow_run_id: "run", + context_version: 4, + }); + f.services.prepare.mockImplementation(async () => { + f.snapshot.job.context_version = 6; + return { job: structuredClone(f.snapshot.job), assetIds: f.snapshot.assets.map(a => a.id) }; + }); + const versions: number[] = []; + f.services.persist.mockImplementation(async (_actor, inputJob, record) => { + const job = inputJob as PackJob; + expect(job.context_version).toBe(f.snapshot.job.context_version); + versions.push(job.context_version!); + f.snapshot.job.context_version!++; + f.snapshot.job.production_context = { + ...(f.snapshot.job.production_context as object), + shared_foundation: structuredClone(record), + }; + return structuredClone(f.snapshot.job); + }); + await runPackFoundationBatch( + actor, + "job", + 1, + { dependentAssetIds: ["video-0"] }, + f.services, + execution, + ); + expect(versions).toEqual([6, 7]); + expect(f.snapshot.job.context_version).toBe(8); + expect(f.services.settle).toHaveBeenCalledWith(actor, "job", undefined, execution); +}); + +it("rejects an intervening context writer after its own pending acknowledgement", async () => { + const f = setup(); + const actor = { ...f.actor, packProtocol: 2 as const }; + const execution = { + expected_revision: 1, + execution_id: "00000000-0000-4000-8000-000000000001", + execution_version: 1, + }; + Object.assign(f.snapshot.job, { + protocol: 2, + ...execution, + revision: 1, + workflow_run_id: "run", + context_version: 4, + }); + f.services.persist.mockImplementation(async (_actor, inputJob, record) => { + expect((inputJob as PackJob).context_version).toBe(f.snapshot.job.context_version); + f.snapshot.job.context_version!++; + f.snapshot.job.production_context = { + ...(f.snapshot.job.production_context as object), + shared_foundation: structuredClone(record), + }; + return structuredClone(f.snapshot.job); + }); + const produce = f.services.produce.getMockImplementation()!; + f.services.produce.mockImplementation(async (...args) => { + await produce(...args); + f.snapshot.job.context_version!++; + f.snapshot.job.production_context = { + ...(f.snapshot.job.production_context as object), + concurrent_writer: "retain this exact context", + }; + }); + await expect( + runPackFoundationBatch( + actor, + "job", + 1, + { dependentAssetIds: ["video-0"] }, + f.services, + execution, + ), + ).rejects.toThrow("Foundation context changed after acknowledgement"); + expect(f.services.persist).toHaveBeenCalledTimes(1); + expect(f.services.bindFiles).not.toHaveBeenCalled(); + expect(f.services.settle).not.toHaveBeenCalled(); + expect(f.produced.map(asset => asset.id)).toEqual(["artwork-0"]); + expect(f.snapshot.job.context_version).toBe(6); + expect(f.snapshot.job.production_context).toHaveProperty( + "concurrent_writer", + "retain this exact context", + ); +}); + +it("rejects a context advance while reusing a saved ready foundation", async () => { + const f = setup(); + await runPackFoundationBatch(f.actor, "job", 1, { dependentAssetIds: ["video-0"] }, f.services); + const actor = { ...f.actor, packProtocol: 2 as const }; + const execution = { + expected_revision: 1, + execution_id: "00000000-0000-4000-8000-000000000001", + execution_version: 1, + }; + Object.assign(f.snapshot.job, { + protocol: 2, + ...execution, + revision: 1, + workflow_run_id: "run", + context_version: 4, + }); + f.services.persist.mockClear(); + f.services.produce.mockClear(); + f.services.bindFiles.mockClear(); + f.services.settle.mockClear(); + let reads = 0; + f.services.read.mockImplementation(async () => { + if (++reads === 2) { + f.snapshot.job.context_version = 5; + f.snapshot.job.production_context = { + ...(f.snapshot.job.production_context as object), + concurrent_writer: "saved-ready update", + }; + } + return structuredClone(f.snapshot); + }); + await expect( + runPackFoundationBatch( + actor, + "job", + 1, + { dependentAssetIds: ["video-0"] }, + f.services, + execution, + ), + ).rejects.toThrow("Foundation context changed after acknowledgement"); + expect(f.services.persist).not.toHaveBeenCalled(); + expect(f.services.produce).not.toHaveBeenCalled(); + expect(f.services.bindFiles).not.toHaveBeenCalled(); + expect(f.services.settle).not.toHaveBeenCalled(); + expect(f.snapshot.job.production_context).toHaveProperty( + "concurrent_writer", + "saved-ready update", + ); +}); diff --git a/lib/content/packs/foundation/__tests__/packFoundationDurablePorts.test.ts b/lib/content/packs/foundation/__tests__/packFoundationDurablePorts.test.ts index 4545bf761..9d5b7f64c 100644 --- a/lib/content/packs/foundation/__tests__/packFoundationDurablePorts.test.ts +++ b/lib/content/packs/foundation/__tests__/packFoundationDurablePorts.test.ts @@ -349,3 +349,27 @@ it("refuses persistence if admission changes after the initial read", async () = expect(mocks.fal).not.toHaveBeenCalled(); expect(mocks.sign).not.toHaveBeenCalled(); }); + +it("settles only the explicitly supplied original foundation execution", async () => { + const actor = { accountId: "owner", packProtocol: 2 as const }; + const execution = { + expected_revision: 1, + execution_id: "00000000-0000-4000-8000-000000000001", + execution_version: 1, + }; + const services = createPackFoundationServices(actor, "job", execution); + expect(() => services.settle(actor, "job")).toThrow("Foundation settlement execution mismatch"); + expect(() => + services.settle(actor, "job", undefined, { ...execution, execution_version: 2 }), + ).toThrow("Foundation settlement execution mismatch"); + expect(mocks.rpc).not.toHaveBeenCalled(); + mocks.rpc.mockResolvedValue({ data: { job: {} }, error: null }); + await services.settle(actor, "job", undefined, execution); + expect(mocks.rpc).toHaveBeenCalledWith( + "release_pack_command_v2", + expect.objectContaining({ + p_action: "settle", + p_input: { ...execution, workflow_run_id: "run", error: null }, + }), + ); +}); diff --git a/lib/content/packs/foundation/__tests__/savePackFoundationRecord.test.ts b/lib/content/packs/foundation/__tests__/savePackFoundationRecord.test.ts index 0a06206bd..66fe332fa 100644 --- a/lib/content/packs/foundation/__tests__/savePackFoundationRecord.test.ts +++ b/lib/content/packs/foundation/__tests__/savePackFoundationRecord.test.ts @@ -72,3 +72,72 @@ it("rejects malformed completed preparation before persisting readiness", async ).rejects.toThrow(); expect(runPackCommand).toHaveBeenCalledTimes(1); }); + +function protocol2Snapshot() { + const snapshot = foundationFixture(); + Object.assign(snapshot.job, { + protocol: 2, + execution_id: "00000000-0000-4000-8000-000000000001", + execution_version: 1, + workflow_run_id: "run", + context_version: 4, + }); + return snapshot; +} +it("refuses a stale foundation record without borrowing a newer context version", async () => { + const original = protocol2Snapshot(); + const record = { + plan: createPackFoundationPlan(original, { dependentAssetIds: [] }), + state: "pending" as const, + files: [], + }; + const current = structuredClone(original); + current.job.context_version = 5; + vi.mocked(runPackCommand).mockResolvedValue(current); + await expect( + savePackFoundationRecord({ accountId: "owner", packProtocol: 2 }, original.job, record), + ).rejects.toThrow(/context.*changed/i); + expect(runPackCommand).toHaveBeenCalledTimes(1); +}); +it("returns acknowledged context versions for consecutive foundation writes", async () => { + const snapshot = protocol2Snapshot(); + const record = { + plan: createPackFoundationPlan(snapshot, { dependentAssetIds: [] }), + state: "pending" as const, + files: [], + }; + vi.mocked(runPackCommand).mockImplementation(async (_actor, action, _id, input) => { + if (action === "context") { + expect(input?.context_version).toBe(snapshot.job.context_version); + snapshot.job.context_version!++; + snapshot.job.production_context = input!.value; + } + return structuredClone(snapshot); + }); + const first = await savePackFoundationRecord( + { accountId: "owner", packProtocol: 2 }, + structuredClone(snapshot.job), + record, + ); + expect(first?.context_version).toBe(5); + const second = await savePackFoundationRecord( + { accountId: "owner", packProtocol: 2 }, + first!, + record, + ); + expect(second?.context_version).toBe(6); +}); + +it("rejects a missing original context version before any persistence call", async () => { + const snapshot = protocol2Snapshot(); + delete snapshot.job.context_version; + const record = { + plan: createPackFoundationPlan(snapshot, { dependentAssetIds: [] }), + state: "pending" as const, + files: [], + }; + await expect( + savePackFoundationRecord({ accountId: "owner", packProtocol: 2 }, snapshot.job, record), + ).rejects.toThrow("Missing original foundation context version"); + expect(runPackCommand).not.toHaveBeenCalled(); +}); diff --git a/lib/content/packs/foundation/packFoundationServices.ts b/lib/content/packs/foundation/packFoundationServices.ts index c3d099955..0224066d9 100644 --- a/lib/content/packs/foundation/packFoundationServices.ts +++ b/lib/content/packs/foundation/packFoundationServices.ts @@ -1,3 +1,4 @@ +import type { PackLaunchFence } from "../fence/packFenceTypes"; import type { PackActor, PackAsset, PackFile, PackJob, PackSnapshot } from "../packTypes"; import type { PackFoundationPlan, @@ -8,7 +9,11 @@ import type { export type PackFoundationServices = { prepare: (actor: PackActor, jobId: string) => Promise<{ job: PackJob; assetIds: string[] }>; read: (actor: PackActor, jobId: string) => Promise; - persist: (actor: PackActor, job: PackJob, record: PackFoundationRecord) => Promise; + persist: ( + actor: PackActor, + job: PackJob, + record: PackFoundationRecord, + ) => Promise; produce: (actor: PackActor, job: PackJob, assetId: string) => Promise; createPlan: ( snapshot: PackSnapshot, @@ -19,7 +24,12 @@ export type PackFoundationServices = { asset: PackAsset | undefined, ) => Promise; signFile: (file: PackFile) => Promise; - settle: (actor: PackActor, jobId: string, error?: string) => Promise; + settle: ( + actor: PackActor, + jobId: string, + error?: string, + execution?: PackLaunchFence, + ) => Promise; }; export type PackFoundationBatchState = { diff --git a/lib/content/packs/foundation/persistPackFoundationContext.ts b/lib/content/packs/foundation/persistPackFoundationContext.ts new file mode 100644 index 000000000..4ce449099 --- /dev/null +++ b/lib/content/packs/foundation/persistPackFoundationContext.ts @@ -0,0 +1,25 @@ +import { assertPackExecutionIdentity } from "../fence/assertPackExecutionIdentity"; +import { getPackLaunchFence } from "../fence/getPackLaunchFence"; +import { PackFenceError } from "../fence/PackFenceError"; +import type { PackActor, PackJob } from "../packTypes"; +import type { PackFoundationRecord } from "./packFoundationSchema"; +import type { PackFoundationServices } from "./packFoundationServices"; +/** Advance only from an acknowledged write, never from a later unrelated read. */ +export async function persistPackFoundationContext( + actor: PackActor, + job: PackJob, + record: PackFoundationRecord, + services: PackFoundationServices, +): Promise { + const { persist } = services; + const saved = await persist(actor, job, record); + if (actor.packProtocol !== 2) return job; + if ( + !saved || + !Number.isSafeInteger(job.context_version) || + saved.context_version !== job.context_version! + 1 + ) + throw new PackFenceError("Foundation context acknowledgement unavailable"); + assertPackExecutionIdentity(saved, getPackLaunchFence(job), job.workflow_run_id ?? undefined); + return saved; +} diff --git a/lib/content/packs/foundation/preparePackFoundationBatch.ts b/lib/content/packs/foundation/preparePackFoundationBatch.ts index f0631911e..a90e51039 100644 --- a/lib/content/packs/foundation/preparePackFoundationBatch.ts +++ b/lib/content/packs/foundation/preparePackFoundationBatch.ts @@ -1,3 +1,5 @@ +import { persistPackFoundationContext } from "./persistPackFoundationContext"; +import { PackFenceError } from "../fence/PackFenceError"; import { resolvePackCreativeContext } from "../resolvePackCreativeContext"; import { bindPackFoundationDependents } from "./bindPackFoundationDependents"; import { getPackFoundationReferences } from "./getPackFoundationReferences"; @@ -45,9 +47,33 @@ export async function preparePackFoundationBatch( prepared.job.quote?.id !== plan.quote_id ) throw new Error("Prepared pack identity changed"); - if (!saved) await persist(actor, original.job, state.foundation); + let acknowledgedJob = prepared.job; + if ( + actor.packProtocol === 2 && + saved && + prepared.job.context_version !== original.job.context_version + ) + throw new PackFenceError("Foundation context changed during preparation"); + if (!saved) { + if ( + actor.packProtocol === 2 && + (prepared.job.production_context as { shared_foundation?: unknown } | null) + ?.shared_foundation !== undefined + ) + throw new PackFenceError("Foundation context changed during preparation"); + acknowledgedJob = await persistPackFoundationContext( + actor, + actor.packProtocol === 2 + ? { ...original.job, context_version: prepared.job.context_version } + : original.job, + state.foundation, + services, + ); + } if (asset?.status === "pending") await produce(actor, prepared.job, asset.id); const current = await read(actor, jobId); + if (actor.packProtocol === 2 && current.job.context_version !== acknowledgedJob.context_version) + throw new PackFenceError("Foundation context changed after acknowledgement"); state.savedJob = current.job; if ((await createPlan(current, selection)).digest !== plan.digest) throw new Error("Foundation plan changed during preparation"); @@ -63,11 +89,19 @@ export async function preparePackFoundationBatch( if (bound.state === "ready") bound.dependency_bindings = bindPackFoundationDependents(current, plan, saved); state.foundation = bound; - await persist(actor, current.job, bound); + const persistedJob = await persistPackFoundationContext( + actor, + actor.packProtocol === 2 + ? { ...current.job, context_version: acknowledgedJob.context_version } + : current.job, + bound, + services, + ); + state.savedJob = persistedJob; const pending = new Set(current.assets.filter(a => a.status === "pending").map(a => a.id)); state.references = await getPackFoundationReferences( actor, - current.job, + persistedJob, prepared.job, bound, plan.dependent_asset_ids.some(id => pending.has(id)), diff --git a/lib/content/packs/foundation/runPackFoundationBatch.ts b/lib/content/packs/foundation/runPackFoundationBatch.ts index aa3ac062f..e8b88b565 100644 --- a/lib/content/packs/foundation/runPackFoundationBatch.ts +++ b/lib/content/packs/foundation/runPackFoundationBatch.ts @@ -1,3 +1,4 @@ +import type { PackLaunchFence } from "../fence/packFenceTypes"; import { getPackFoundationContinuation } from "./getPackFoundationContinuation"; import { preparePackFoundationBatch } from "./preparePackFoundationBatch"; import { runBoundedPackProduction } from "../runBoundedPackProduction"; @@ -11,6 +12,7 @@ export async function runPackFoundationBatch( concurrency: number, selection: PackFoundationSelection, services: PackFoundationServices, + execution?: PackLaunchFence, ) { const { produce, settle } = services; const state: PackFoundationBatchState = { @@ -18,27 +20,34 @@ export async function runPackFoundationBatch( settlementDeferred: true, references: [], }; - const result = await runBoundedPackProduction(actor, jobId, concurrency, { - prepare: () => preparePackFoundationBatch(actor, jobId, selection, services, state), - produce: async (stepActor, job, assetId) => { - if (state.foundation?.plan.dependent_asset_ids.includes(assetId)) { - if (state.foundation.state !== "ready") - throw new Error("Foundation dependency is not ready"); - job = { - ...job, - brief: { ...job.brief, reference_image_urls: state.references }, - shared_foundation_binding: { - plan_sha256: state.foundation.plan.digest, - asset_sha256: state.foundation.asset_sha256, - }, - }; - } - return produce(stepActor, job, assetId); + const result = await runBoundedPackProduction( + actor, + jobId, + concurrency, + { + prepare: () => preparePackFoundationBatch(actor, jobId, selection, services, state), + produce: async (stepActor, job, assetId) => { + if (state.foundation?.plan.dependent_asset_ids.includes(assetId)) { + if (state.foundation.state !== "ready") + throw new Error("Foundation dependency is not ready"); + job = { + ...job, + brief: { ...job.brief, reference_image_urls: state.references }, + shared_foundation_binding: { + plan_sha256: state.foundation.plan.digest, + asset_sha256: state.foundation.asset_sha256, + }, + }; + } + return produce(stepActor, job, assetId); + }, + settle: async (stepActor, id, error, fence) => { + if (!state.settlementDeferred) + await settle(stepActor, id, error, ...(stepActor.packProtocol === 2 ? [fence] : [])); + }, }, - settle: async (stepActor, id, error) => { - if (!state.settlementDeferred) await settle(stepActor, id, error); - }, - }); + execution, + ); const continuation = getPackFoundationContinuation( state.foundation, state.savedJob, diff --git a/lib/content/packs/foundation/savePackFoundationRecord.ts b/lib/content/packs/foundation/savePackFoundationRecord.ts index a08740142..dfb7b15e7 100644 --- a/lib/content/packs/foundation/savePackFoundationRecord.ts +++ b/lib/content/packs/foundation/savePackFoundationRecord.ts @@ -1,3 +1,5 @@ +import { PackFenceError } from "../fence/PackFenceError"; +import { getPackExecutionFence } from "../fence/getPackExecutionFence"; import { packProductionContextSchema } from "../packProductionContextSchema"; import { runPackCommand } from "@/lib/supabase/release_pack_jobs/runPackCommand"; import { createPackFoundationPlan } from "./createPackFoundationPlan"; @@ -10,7 +12,14 @@ export async function savePackFoundationRecord( input: PackFoundationRecord, ) { const record = packFoundationRecordSchema.parse(input); + if ( + job.protocol === 2 && + (!Number.isSafeInteger(job.context_version) || job.context_version! < 0) + ) + throw new PackFenceError("Missing original foundation context version"); const current = await runPackCommand(actor, "read", job.id); + if (job.protocol === 2 && current.job.context_version !== job.context_version) + throw new PackFenceError("Foundation context version changed"); const plan = createPackFoundationPlan(current, { ...(record.plan.selection === "explicit" ? { foundationAssetId: record.plan.asset_id! } : {}), dependentAssetIds: record.plan.dependent_asset_ids, @@ -18,6 +27,10 @@ export async function savePackFoundationRecord( if ( current.job.status !== "running" || current.job.owner_id !== job.owner_id || + (job.protocol === 2 && + (current.job.execution_id !== job.execution_id || + current.job.execution_version !== job.execution_version || + current.job.workflow_run_id !== job.workflow_run_id)) || plan.digest !== record.plan.digest ) throw new Error("Foundation quote, direction or admitted job changed"); @@ -29,7 +42,10 @@ export async function savePackFoundationRecord( !(record.blocked_reason === "reference_limit" && record.state === "pending")) ) throw new Error("Persist foundation only after retained shared preparation completes"); - await runPackCommand(actor, "context", job.id, { + const saved = await runPackCommand(actor, "context", job.id, { + ...getPackExecutionFence(job), + ...(job.protocol === 2 ? { context_version: job.context_version } : {}), value: { ...context, shared_foundation: record }, }); + return saved.job; } diff --git a/lib/content/packs/packTypes.ts b/lib/content/packs/packTypes.ts index 19da137a6..009fe59c1 100644 --- a/lib/content/packs/packTypes.ts +++ b/lib/content/packs/packTypes.ts @@ -17,6 +17,7 @@ export type PackAsset = { ordinal: number; status: "pending" | "running" | "completed" | "failed" | "uncertain"; attempt: number; + claim_id?: string; completed_at?: string | null; credits: number; maximum_usd: number; @@ -32,6 +33,10 @@ export type PackJob = { owner_id: string; created_by: string; revision: number; + protocol?: 1 | 2; + execution_id?: string; + execution_version?: number; + context_version?: number; status: string; brief: ReleasePackBrief; deliverables: ReleasePackDeliverables; @@ -46,6 +51,11 @@ export type PackSnapshot = { assets: PackAsset[]; dispatch?: boolean; claimed?: boolean; + replayed?: boolean; asset?: PackAsset; }; -export type PackActor = { accountId: string; organizationId?: string }; +export type PackActor = { + accountId: string; + organizationId?: string; + /** Internal opt-in; no public route populates this. */ packProtocol?: 2; +}; diff --git a/lib/content/packs/preparePackProduction.ts b/lib/content/packs/preparePackProduction.ts index 87ad95ba2..532508ee3 100644 --- a/lib/content/packs/preparePackProduction.ts +++ b/lib/content/packs/preparePackProduction.ts @@ -1,3 +1,4 @@ +import { getPackExecutionFence } from "./fence/getPackExecutionFence"; import { packProductionContextSchema as contextSchema } from "./packProductionContextSchema"; import { parsePackAudioAnalysis } from "./parsePackAudioAnalysis"; import { assertPackQuoteConfiguration } from "./assertPackQuoteConfiguration"; @@ -73,9 +74,12 @@ export async function preparePackProduction(actor: PackActor, job: PackJob): Pro throw new Error("Selected song excerpt runs past the supplied recording"); const audio = { ...sources.audio, url: audioUrl }; const references = sources.references; - await runPackCommand(actor, "context", job.id, { + const analyzing = await runPackCommand(actor, "context", job.id, { + ...getPackExecutionFence(job), + ...(job.protocol === 2 ? { context_version: job.context_version } : {}), value: { state: "analyzing", audio, references, duration_seconds: duration }, }); + if (actor.packProtocol === 2) job = { ...job, context_version: analyzing.job.context_version }; assertPackQuoteConfiguration(job.quote); const analysis = await callFlamingoGenerate( flamingoGenerateBodySchema.parse({ @@ -100,7 +104,12 @@ export async function preparePackProduction(actor: PackActor, job: PackJob): Pro selected_start_seconds: job.brief.audio_start_seconds ?? interpreted.proposed_start_seconds, duration_seconds: duration, }; - await runPackCommand(actor, "context", job.id, { value }); + const completed = await runPackCommand(actor, "context", job.id, { + ...getPackExecutionFence(job), + ...(job.protocol === 2 ? { context_version: job.context_version } : {}), + value, + }); + if (actor.packProtocol === 2) job = { ...job, context_version: completed.job.context_version }; context = contextSchema.safeParse(value); } if (!context.success) throw new Error("Production context unavailable"); diff --git a/lib/content/packs/processPackOperation.ts b/lib/content/packs/processPackOperation.ts index 75e62531d..9151aa72a 100644 --- a/lib/content/packs/processPackOperation.ts +++ b/lib/content/packs/processPackOperation.ts @@ -1,3 +1,6 @@ +import type { PackProtocol2Admission } from "./fence/packFenceTypes"; +import { getPackTransitionInput } from "./fence/getPackTransitionInput"; +import { getPackLaunchFence } from "./fence/getPackLaunchFence"; import { assertPackQuoteConfiguration } from "./assertPackQuoteConfiguration"; import { importPackSources } from "./importPackSources"; import { randomUUID, createHash } from "node:crypto"; @@ -38,6 +41,7 @@ export async function processPackOperation( operation: PackOperation, raw: unknown, credentialOrganizationId?: string | null, + protocol2?: PackProtocol2Admission, ) { const input = packOperationSchemas[operation].parse(raw) as z.infer< typeof prepareReleasePackSchema @@ -46,7 +50,11 @@ export async function processPackOperation( const organizationId = input.organization_id ?? credentialOrganizationId ?? undefined; if (organizationId && !(await validateOrganizationAccess({ accountId, organizationId }))) throw new Error("Pack workspace access denied"); - const actor: PackActor = { accountId, organizationId }; + const actor: PackActor = { + accountId, + organizationId, + ...(protocol2 ? { packProtocol: 2 as const } : {}), + }; if (operation === "upload") return uploadPackAsset(actor, raw); if (operation === "prepare") { if (input.job_id && !input.expected_revision) @@ -74,7 +82,12 @@ export async function processPackOperation( const saved = await runPackCommand(actor, "read", input.job_id!); if (operation === "get") return readPackSnapshot(saved); if (operation === "export") { - const marked = await runPackCommand(actor, "export", saved.job.id); + const marked = await runPackCommand( + actor, + "export", + saved.job.id, + protocol2 ? { expected_revision: saved.job.revision } : {}, + ); return { ...(await readPackSnapshot(marked)), manifest: { @@ -143,6 +156,12 @@ export async function processPackOperation( await runPackCommand(actor, "estimate", saved.job.id, { quote, assets: plan, + ...(protocol2 + ? { + expected_revision: saved.job.revision, + ...(selected ? { asset_attempt: selected.attempt } : {}), + } + : {}), ...(selected ? { retry_asset_id: selected.id } : {}), }), ); @@ -177,6 +196,7 @@ export async function processPackOperation( "Submission outcome is uncertain; no safe automatic retry is available. Your existing outputs and brief are retained.", ); funded = await runPackCommand(actor, "resume", saved.job.id, { + ...getPackTransitionInput(protocol2), asset_id: selected.id, quote_id: input.quote_id, expected_revision: input.expected_revision, @@ -187,7 +207,10 @@ export async function processPackOperation( assertPackQuoteConfiguration(saved.job.quote); if (saved.job.quote.retry_asset_id !== input.asset_id) throw new Error("Request and approve a fresh estimate for this failed asset first"); - funded = await runPackCommand(actor, "approve", saved.job.id, input); + funded = await runPackCommand(actor, "approve", saved.job.id, { + ...input, + ...getPackTransitionInput(protocol2), + }); } } else { assertPackQuoteConfiguration(saved.job.quote); @@ -200,16 +223,29 @@ export async function processPackOperation( }); if (credits.kind !== "available") throw new Error("Insufficient credits; your brief and quote are saved"); - funded = await runPackCommand(actor, "approve", saved.job.id, input); + funded = await runPackCommand(actor, "approve", saved.job.id, { + ...input, + ...getPackTransitionInput(protocol2), + }); } if (funded.dispatch) { try { const { start } = await import("workflow/api"); const { releasePackWorkflow } = await import("@/app/workflows/packs/releasePackWorkflow"); - const run = await start(releasePackWorkflow, [actor, saved.job.id, resumedAssetId]); - funded = await runPackCommand(actor, "run", saved.job.id, { workflow_run_id: run.runId }); + const execution = protocol2 ? getPackLaunchFence(funded.job) : undefined; + const run = await start( + releasePackWorkflow, + execution + ? [actor, saved.job.id, resumedAssetId, execution] + : [actor, saved.job.id, resumedAssetId], + ); + funded = await runPackCommand(actor, "run", saved.job.id, { + ...execution, + workflow_run_id: run.runId, + }); } catch { funded = await runPackCommand(actor, "dispatch_uncertain", saved.job.id, { + ...(protocol2 ? getPackLaunchFence(funded.job) : {}), error: "Workflow dispatch outcome is uncertain. Its budget remains held while accepted work is reconciled. Read this job; do not resubmit.", }); diff --git a/lib/content/packs/runBoundedPackProduction.ts b/lib/content/packs/runBoundedPackProduction.ts index a3541f075..6db62312b 100644 --- a/lib/content/packs/runBoundedPackProduction.ts +++ b/lib/content/packs/runBoundedPackProduction.ts @@ -1,9 +1,28 @@ +import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes"; +import { assertPackExecutionIdentity } from "@/lib/content/packs/fence/assertPackExecutionIdentity"; +import { isPackFenceError } from "@/lib/content/packs/fence/isPackFenceError"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; import type { PackActor, PackJob } from "./packTypes"; type PackBatchServices = { - prepare: (actor: PackActor, jobId: string) => Promise<{ job: PackJob; assetIds: string[] }>; - produce: (actor: PackActor, job: PackJob, assetId: string) => Promise; - settle: (actor: PackActor, jobId: string, error?: string) => Promise; + prepare: ( + actor: PackActor, + jobId: string, + execution?: PackLaunchFence, + ) => Promise<{ job: PackJob; assetIds: string[]; assetAttempts?: Record }>; + produce: ( + actor: PackActor, + job: PackJob, + assetId: string, + resume?: boolean, + expectedAttempt?: number, + ) => Promise; + settle: ( + actor: PackActor, + jobId: string, + error?: string, + execution?: PackLaunchFence, + ) => Promise; }; /** @@ -16,24 +35,33 @@ export async function runBoundedPackProduction( jobId: string, concurrency: number, services: PackBatchServices, + execution?: PackLaunchFence, ): Promise<{ job_id: string; error?: string; failed_asset_ids?: string[] }> { if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error("Pack concurrency must be a positive safe integer"); + if (actor.packProtocol === 2 && !execution) + throw new PackFenceError("Scheduler requires original execution admission"); // Workflow serializes a method call receiver; never pass the services object as this. const { prepare, produce, settle } = services; const failedAssetIds: string[] = []; let stopped = false; let failure: unknown; try { - const prepared = await prepare(actor, jobId); + const prepared = + actor.packProtocol === 2 + ? await prepare(actor, jobId, execution) + : await prepare(actor, jobId); + if (actor.packProtocol === 2) assertPackExecutionIdentity(prepared.job, execution); const assetIds = [...new Set(prepared.assetIds)]; let next = 0; async function worker() { while (!stopped && next < assetIds.length) { const assetId = assetIds[next++]; try { - await produce(actor, prepared.job, assetId); + if (actor.packProtocol === 2) + await produce(actor, prepared.job, assetId, false, prepared.assetAttempts?.[assetId]); + else await produce(actor, prepared.job, assetId); } catch (error) { failedAssetIds.push(assetId); if (!stopped) failure = error; @@ -48,11 +76,13 @@ export async function runBoundedPackProduction( failure = error; } + if ((actor.packProtocol === 2 && stopped) || isPackFenceError(failure)) throw failure; // Outside the catch: a lost settlement acknowledgement must not invoke it again. await settle( actor, jobId, stopped ? (failure instanceof Error ? failure.message : "Pack workflow failed") : undefined, + ...(actor.packProtocol === 2 && execution ? [execution] : []), ); return stopped ? { diff --git a/lib/content/packs/runPackFalRequest.ts b/lib/content/packs/runPackFalRequest.ts index 948e7f42c..6647fc9d2 100644 --- a/lib/content/packs/runPackFalRequest.ts +++ b/lib/content/packs/runPackFalRequest.ts @@ -1,3 +1,5 @@ +import { PackFenceError } from "./fence/PackFenceError"; +import type { PackProviderReceipt } from "./fence/packFenceTypes"; import fal from "@/lib/fal/server"; import { createHash } from "node:crypto"; /** Persist submission intent and accepted IDs before polling; ambiguous submits are never repeated. */ @@ -8,20 +10,34 @@ export async function runPackFalRequest(input: { key: string; state: Record; save: (key: string, value: unknown) => Promise; + observe?: (receipt: PackProviderReceipt) => Promise; }) { - type Receipt = { - state: string; - request_id?: string; - data?: unknown; - charge_credits?: number; - model?: string; - input_sha256?: string; - }; + type Receipt = PackProviderReceipt; let receipt = input.state[input.key] as Receipt | undefined; const save = async (value: Receipt) => { - await input.save(input.key, value); - input.state[input.key] = value; - receipt = value; + if (input.observe && (value.state === "accepted" || value.state === "complete")) + await input.observe(value); + const result = await input.save(input.key, value); + if ( + input.observe && + value.state === "submitting" && + (!result || + typeof result !== "object" || + !("operation_created" in result) || + result.operation_created !== true) + ) + throw new PackFenceError( + "Submission intent already recorded; reconcile without resubmission", + ); + const operationId = + result && + typeof result === "object" && + "operation_id" in result && + typeof result.operation_id === "string" + ? result.operation_id + : value.operation_id; + receipt = operationId ? { ...value, operation_id: operationId } : value; + input.state[input.key] = receipt; }; if (receipt?.state === "complete") return receipt.data as Record; if (receipt && !receipt.request_id) diff --git a/lib/supabase/release_pack_jobs/__tests__/packFenceAdapters.test.ts b/lib/supabase/release_pack_jobs/__tests__/packFenceAdapters.test.ts new file mode 100644 index 000000000..fab23a17f --- /dev/null +++ b/lib/supabase/release_pack_jobs/__tests__/packFenceAdapters.test.ts @@ -0,0 +1,52 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { readPackProviderObservations } from "../readPackProviderObservations"; +import { runPackCommand } from "../runPackCommand"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; +const rpc = vi.hoisted(() => vi.fn()); +vi.mock("../../serverClient", () => ({ default: { rpc } })); +const id = "00000000-0000-4000-8000-000000000001"; +const actor = { accountId: id, packProtocol: 2 as const }; +beforeEach(() => rpc.mockReset()); +describe("redacted receipt reads", () => { + it.each([ + null, + { operations: [{ secret: "must not escape" }], next_cursor: null }, + { operations: [], next_cursor: "AAAAAAAA-0000-4000-8000-000000000001" }, + ])("rejects malformed/unredacted pages without leaking details", async data => { + rpc.mockResolvedValue({ data, error: null }); + await expect(readPackProviderObservations(actor, id)).rejects.toBeInstanceOf(PackFenceError); + }); + it("accepts empty page", async () => { + const data = { operations: [], next_cursor: null }; + rpc.mockResolvedValue({ data, error: null }); + expect(await readPackProviderObservations(actor, id)).toEqual(data); + }); +}); +describe("command failure isolation", () => { + it.each([1, 2] as const)("preserves protocol %i failure contract", async protocol => { + const selected = protocol === 2 ? actor : { accountId: id }; + rpc.mockResolvedValue({ + data: null, + error: { code: "P0002", message: "Insufficient credits" }, + }); + await expect(runPackCommand(selected, "approve", id)).rejects.toThrow( + protocol === 2 ? "reconciliation" : "Insufficient credits", + ); + expect(rpc.mock.calls[0][0]).toBe( + protocol === 2 ? "release_pack_command_v2" : "release_pack_command", + ); + const original = new Error("transport secret"); + rpc.mockRejectedValue(original); + const failed = runPackCommand(selected, "settle", id); + if (protocol === 2) await expect(failed).rejects.toBeInstanceOf(PackFenceError); + else await expect(failed).rejects.toBe(original); + rpc.mockResolvedValue({ data: {}, error: null }); + await expect(runPackCommand(selected, "read", id)).rejects.toThrow( + protocol === 2 ? "Invalid protocol-2" : "Invalid pack", + ); + rpc.mockResolvedValue({ data: { job: {}, command_expired: true }, error: null }); + await expect(runPackCommand(selected, "settle", id)).rejects.toThrow( + protocol === 2 ? "reconcile retained" : "inspect saved", + ); + }); +}); diff --git a/lib/supabase/release_pack_jobs/appendPackProviderObservation.ts b/lib/supabase/release_pack_jobs/appendPackProviderObservation.ts new file mode 100644 index 000000000..2858479e7 --- /dev/null +++ b/lib/supabase/release_pack_jobs/appendPackProviderObservation.ts @@ -0,0 +1,38 @@ +import supabase from "../serverClient"; +import { z } from "zod"; +import type { PackActor } from "@/lib/content/packs/packTypes"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; +import { packObservationPayloadSchema } from "./packObservationPayloadSchema"; +/** Separate RPC transaction: append acknowledgement precedes the active phase write. */ +export async function appendPackProviderObservation( + actor: PackActor, + operationId: string, + identity: { execution_id?: string; claim_id?: string }, + payload: unknown, +) { + if (actor.packProtocol !== 2) + throw new PackFenceError("Receipt port requires explicit protocol 2"); + const parsed = packObservationPayloadSchema.safeParse(payload); + const target = z.strictObject({ execution_id: z.uuid(), claim_id: z.uuid() }).safeParse(identity); + if (!parsed.success || !target.success || !z.uuid().safeParse(operationId).success) + throw new PackFenceError("Invalid redacted receipt payload or identity"); + const rpc = supabase.rpc.bind(supabase) as unknown as ( + name: string, + input: Record, + ) => PromiseLike<{ data: unknown; error: unknown }>; + const { data, error } = await Promise.resolve( + rpc("append_pack_provider_observation", { + p_actor: actor.accountId, + p_org: actor.organizationId ?? null, + p_operation: operationId, + p_identity: target.data, + p_payload: parsed.data, + }), + ).catch(() => { + throw new PackFenceError("Receipt persistence uncertain"); + }); + const result = z.object({ state: z.enum(["saved", "reused"]) }).safeParse(data); + if (error || !result.success) + throw new PackFenceError("Provider receipt requires reconciliation; no phase was advanced"); + return result.data; +} diff --git a/lib/supabase/release_pack_jobs/packObservationPayloadSchema.ts b/lib/supabase/release_pack_jobs/packObservationPayloadSchema.ts new file mode 100644 index 000000000..bcd3c7483 --- /dev/null +++ b/lib/supabase/release_pack_jobs/packObservationPayloadSchema.ts @@ -0,0 +1,20 @@ +import { z } from "zod"; +export const packObservationPayloadSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("accepted"), + request_id: z.string().regex(/^[A-Za-z0-9_-]{1,200}$/), + }), + z.strictObject({ + kind: z.literal("completed"), + request_id: z.string().regex(/^[A-Za-z0-9_-]{1,200}$/), + output_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }), + z.strictObject({ + kind: z.literal("failed"), + request_id: z.string().regex(/^[A-Za-z0-9_-]{1,200}$/), + reason_code: z.enum(["provider_failed", "response_invalid", "result_unavailable"]), + }), +]); diff --git a/lib/supabase/release_pack_jobs/packReceiptPageSchema.ts b/lib/supabase/release_pack_jobs/packReceiptPageSchema.ts new file mode 100644 index 000000000..80d014a71 --- /dev/null +++ b/lib/supabase/release_pack_jobs/packReceiptPageSchema.ts @@ -0,0 +1,46 @@ +import { z } from "zod"; +import { packObservationPayloadSchema } from "./packObservationPayloadSchema"; +const uuid = z.uuid().refine(value => value === value.toLowerCase()); +const hash = z.string().regex(/^[a-f0-9]{64}$/); +const timestamp = z.iso.datetime({ offset: true }); +const conflict = z.strictObject({ + kind: z.enum(["accepted", "completed", "failed"]), + existing_sha256: hash, + incoming_sha256: hash, +}); +const observation = z + .strictObject({ + operation_id: uuid, + kind: z.enum(["accepted", "completed", "failed", "conflict"]), + payload: z.union([packObservationPayloadSchema, conflict]), + recorded_at: timestamp, + }) + .refine(row => + row.kind === "conflict" ? "existing_sha256" in row.payload : row.kind === row.payload.kind, + ); +export const packReceiptPageSchema = z.strictObject({ + operations: z + .array( + z.strictObject({ + id: uuid, + owner_id: uuid, + job_id: uuid, + asset_id: uuid, + revision: z.number().int().positive(), + execution_id: uuid, + execution_version: z.number().int().positive(), + workflow_run_id: z.string().min(1).max(200), + claim_id: uuid, + attempt: z.number().int().positive(), + phase: z.string().min(1).max(100), + provider: z.literal("fal"), + model: z.string().min(1).max(200), + input_sha256: hash, + charge_credits: z.number().int().positive(), + created_at: timestamp, + observations: z.array(observation).max(4), + }), + ) + .max(100), + next_cursor: uuid.nullable(), +}); diff --git a/lib/supabase/release_pack_jobs/readPackProviderObservations.ts b/lib/supabase/release_pack_jobs/readPackProviderObservations.ts new file mode 100644 index 000000000..18509ee3a --- /dev/null +++ b/lib/supabase/release_pack_jobs/readPackProviderObservations.ts @@ -0,0 +1,41 @@ +import supabase from "../serverClient"; +import { z } from "zod"; +import type { PackActor } from "@/lib/content/packs/packTypes"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; +import { packReceiptPageSchema } from "./packReceiptPageSchema"; +/** Internal owner-scoped cursor read. Not mounted on a public route. */ +export async function readPackProviderObservations( + actor: PackActor, + jobId: string, + after: string | null = null, + limit = 100, +) { + if ( + actor.packProtocol !== 2 || + !z.uuid().safeParse(jobId).success || + (after !== null && !z.uuid().safeParse(after).success) || + !Number.isInteger(limit) || + limit < 1 || + limit > 100 + ) + throw new PackFenceError("Invalid receipt page"); + const rpc = supabase.rpc.bind(supabase) as unknown as ( + name: string, + input: Record, + ) => PromiseLike<{ data: unknown; error: unknown }>; + const { data, error } = await Promise.resolve( + rpc("read_pack_provider_observations", { + p_actor: actor.accountId, + p_org: actor.organizationId ?? null, + p_job: jobId, + p_after: after, + p_limit: limit, + }), + ).catch(() => { + throw new PackFenceError("Receipt read unavailable"); + }); + if (error) throw new PackFenceError("Receipt read unavailable"); + const result = packReceiptPageSchema.safeParse(data); + if (!result.success) throw new PackFenceError("Invalid redacted receipt response"); + return result.data; +} diff --git a/lib/supabase/release_pack_jobs/runPackCommand.ts b/lib/supabase/release_pack_jobs/runPackCommand.ts index b01643fb7..1dbd12ba5 100644 --- a/lib/supabase/release_pack_jobs/runPackCommand.ts +++ b/lib/supabase/release_pack_jobs/runPackCommand.ts @@ -1,3 +1,4 @@ +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; import supabase from "../serverClient"; import type { PackActor, PackSnapshot } from "@/lib/content/packs/packTypes"; /** Narrow service-only RPC adapter until generated database types follow migration release. */ @@ -11,13 +12,19 @@ export async function runPackCommand( name: string, args: Record, ) => PromiseLike<{ data: unknown; error: { message: string; code?: string } | null }>; - const { data, error } = await rpc("release_pack_command", { - p_actor: actor.accountId, - p_org: actor.organizationId ?? null, - p_action: action, - p_job: jobId, - p_input: input, + const { data, error } = await Promise.resolve( + rpc(actor.packProtocol === 2 ? "release_pack_command_v2" : "release_pack_command", { + p_actor: actor.accountId, + p_org: actor.organizationId ?? null, + p_action: action, + p_job: jobId, + p_input: input, + }), + ).catch(error => { + if (actor.packProtocol === 2) throw new PackFenceError(); + throw error; }); + if (error && actor.packProtocol === 2) throw new PackFenceError(); if (error) throw new Error( error.code === "P0002" @@ -26,8 +33,12 @@ export async function runPackCommand( : "Pack not found" : error.message, ); - if (!data || typeof data !== "object" || !("job" in data)) + if (!data || typeof data !== "object" || !("job" in data)) { + if (actor.packProtocol === 2) throw new PackFenceError("Invalid protocol-2 storage response"); throw new Error("Invalid pack storage response"); + } + if (actor.packProtocol === 2 && "command_expired" in data && data.command_expired) + throw new PackFenceError("Pack execution expired; reconcile retained work"); if ("command_expired" in data && data.command_expired) throw new Error("Pack production window expired; inspect saved status before retrying"); return data as PackSnapshot; diff --git a/scripts/pack-fence-postgres/run.py b/scripts/pack-fence-postgres/run.py new file mode 100644 index 000000000..5be17b828 --- /dev/null +++ b/scripts/pack-fence-postgres/run.py @@ -0,0 +1,55 @@ +"""Explicit source-bound local paired test. No hosted credentials or TCP listener.""" +import argparse +import hashlib +import json +import os +import pathlib +import subprocess +import sys + +parser = argparse.ArgumentParser() +parser.add_argument('--database', required=True, type=pathlib.Path) +parser.add_argument('--expected-binding', help='SHA256 from --print-binding; binds HEAD and file bytes, including dirty candidates') +parser.add_argument('--print-binding', action='store_true', help='Print source receipt without importing or running it') +args = parser.parse_args() +database = args.database.resolve() +proposal = database / 'proposals/pack-worker-fence-2269' +paths = sorted([*proposal.glob('*.py'), *proposal.glob('*.sql'), *database.glob('tests/*.py'), + database / 'supabase/migrations/20260827050000_add_usage_events_resource_url.sql', + database / 'supabase/migrations/20261020150000_release_pack_jobs.sql']) +assert all((proposal / name).is_file() for name in ['test_contract.py', 'schema.sql', 'command.sql', 'receipts.sql']) + +def binding(): + head = subprocess.check_output(['git', '-C', str(database), 'rev-parse', 'HEAD'], text=True).strip() + files = {str(path.relative_to(database)): hashlib.sha256(path.read_bytes()).hexdigest() for path in paths} + receipt = {'head': head, 'files': files} + digest = hashlib.sha256(json.dumps(receipt, sort_keys=True).encode()).hexdigest() + return {**receipt, 'binding': digest} + +receipt = binding() +if args.print_binding: + print(json.dumps(receipt, indent=2)) + raise SystemExit(0) +if args.expected_binding != receipt['binding']: + parser.error('Expected source binding missing or changed; inspect --print-binding and supply its binding') +print(json.dumps(receipt, sort_keys=True), flush=True) +# All fixture subprocesses, including initdb/psql, inherit only these local runtime settings. +env = {key: value for key, value in os.environ.items() if key in + {'PATH', 'HOME', 'USER', 'LOGNAME', 'TMPDIR', 'TMP', 'TEMP', 'LANG', 'LC_ALL', 'SYSTEMROOT'}} +env['PYTHONDONTWRITEBYTECODE'] = '1' +os.environ.clear() +os.environ.update(env) +sys.path.insert(0, str(proposal)) +from test_contract import FencedPacks +root = pathlib.Path(__file__).resolve().parents[2] +try: + FencedPacks.setUpClass() + env.update(PACK_FENCE_TEST_SOCKET=FencedPacks.tmp.name, PACK_FENCE_TEST_PORT='55476') + result = subprocess.run(['pnpm', 'exec', 'vitest', 'run', 'lib/content/packs/fence/__tests__/packFencePostgres.test.ts'], cwd=root, env=env) + if binding() != receipt: + raise RuntimeError('Database source changed during qualification') + raise SystemExit(result.returncode) +finally: + FencedPacks.doClassCleanups() + if FencedPacks.tearDown_exceptions: + raise RuntimeError('Disposable PostgreSQL cleanup was not confirmed') diff --git a/scripts/pack-fence-workflow-fixture/run.mjs b/scripts/pack-fence-workflow-fixture/run.mjs new file mode 100644 index 000000000..1dfaafed7 --- /dev/null +++ b/scripts/pack-fence-workflow-fixture/run.mjs @@ -0,0 +1,284 @@ +/** Actual compiled production workflow/steps, synthetic effects, zero network. */ +import http2 from "node:http2"; +import { syncBuiltinESMExports } from "node:module"; +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { + mkdtemp, + realpath, + mkdir, + readFile, + writeFile, + readdir, + symlink, + rm, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve, dirname } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { createHash } from "node:crypto"; +import { denyNetwork } from "../pack-scheduler-workflow-fixture/denyNetwork.mjs"; +import { createFixtureWorld } from "../pack-scheduler-workflow-fixture/createFixtureWorld.mjs"; +import { waitFor } from "../pack-scheduler-workflow-fixture/waitFor.mjs"; +const require = createRequire(import.meta.url), + here = dirname(fileURLToPath(import.meta.url)), + root = resolve(here, "../.."); +const network = denyNetwork(), + temp = await realpath(await mkdtemp(join(tmpdir(), "pack-fence-sdk-"))), + bindings = []; +// Explicitly deny additional transports as well as the shared underlying socket guard. +let additionalNetworkAttempts = 0; +const denyAdditionalTransport = () => { + additionalNetworkAttempts++; + throw new Error("Network forbidden in Pack fence fixture"); +}; +globalThis.WebSocket = class { + constructor() { + denyAdditionalTransport(); + } +}; +http2.connect = denyAdditionalTransport; +syncBuiltinESMExports(); +assert.throws(() => new WebSocket("wss://example.invalid"), /Network forbidden/); +assert.throws(() => http2.connect("https://example.invalid"), /Network forbidden/); +assert.equal(additionalNetworkAttempts, 2); +additionalNetworkAttempts = 0; +const source = join(temp, "source"); +await mkdir(source); +await symlink(join(root, "node_modules"), join(temp, "node_modules"), "dir"); +const paths = { "@/*": [join(root, "*")] }; +try { + for (const name of [ + "preparePackStep", + "producePackAssetStep", + "settlePackStep", + "settleBoundedPackStep", + "releasePackWorkflow", + ]) { + const path = `app/workflows/packs/${name}.ts`, + bytes = await readFile(join(root, path)); + bindings.push({ path, sha256: createHash("sha256").update(bytes).digest("hex") }); + await writeFile( + join(source, name === "releasePackWorkflow" ? "workflow.ts" : `${name}.ts`), + name === "releasePackWorkflow" + ? bytes + .toString() + .replace("function releasePackWorkflow(", "function packSchedulerFixture(") + : bytes, + ); + } + for (const file of await readdir(join(here, "source"))) { + const bytes = await readFile(join(here, "source", file)); + bindings.push({ + path: `scripts/pack-fence-workflow-fixture/source/${file}`, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + await writeFile(join(source, file.slice(0, -4)), bytes); + } + for (const path of [ + "pnpm-lock.yaml", + "scripts/pack-fence-workflow-fixture/run.mjs", + "scripts/pack-scheduler-workflow-fixture/createFixtureWorld.mjs", + "scripts/pack-scheduler-workflow-fixture/denyNetwork.mjs", + "scripts/pack-scheduler-workflow-fixture/waitFor.mjs", + ...(await readdir(join(root, "lib/content/packs/fence"))) + .filter(name => name.endsWith(".ts")) + .map(name => `lib/content/packs/fence/${name}`), + ]) { + const bytes = await readFile(join(root, path)); + bindings.push({ path, sha256: createHash("sha256").update(bytes).digest("hex") }); + } + for (const module of [ + "@/lib/supabase/release_pack_jobs/runPackCommand", + "@/lib/content/packs/preparePackProduction", + "@/lib/content/packs/createPackProductionServices", + ]) + paths[module] = [join(source, module.split("/").at(-1) + ".ts")]; + await writeFile( + join(source, "tsconfig.json"), + JSON.stringify({ + compilerOptions: { target: "ES2022", module: "ESNext", moduleResolution: "bundler", paths }, + }), + ); + const builders = createRequire(require.resolve("workflow/next")); + const { StandaloneBuilder } = await import(pathToFileURL(builders.resolve("@workflow/builders"))); + const old = process.cwd(); + try { + process.chdir(source); + await new StandaloneBuilder({ + buildTarget: "standalone", + workingDir: source, + projectRoot: source, + dirs: ["."], + stepsBundlePath: "../generated/steps.cjs", + workflowsBundlePath: "../generated/workflows.cjs", + webhookBundlePath: "../generated/webhook.cjs", + clientBundlePath: join(temp, "generated/client.mjs"), + }).build(); + } finally { + process.chdir(old); + } + const results = []; + for (const scenario of ["phase-failure", "settlement-failure"]) { + const effects = join(temp, scenario); + await mkdir(effects); + const jobId = "00000000-0000-4000-8000-000000000001", + executionId = "00000000-0000-4000-8000-000000000002"; + const job = { + id: jobId, + owner_id: "fixture", + created_by: "fixture", + revision: 1, + protocol: 2, + execution_id: executionId, + execution_version: 1, + status: "running", + held_credits: 100, + consumed_credits: 0, + workflow_run_id: null, + error: null, + quote: null, + brief: { + artist_name: "Fixture", + song_title: "Fixture", + primary_action: "listen", + allow_creative_defaults: false, + reference_image_urls: [], + platforms: ["instagram"], + evidence: [], + constraints: [], + audio_url: "https://example.invalid/synthetic", + }, + deliverables: { + videos: 0, + quote_cards: 1, + artwork: 0, + carousels: 0, + visualizers: 0, + video_seconds: 8, + }, + }; + await writeFile( + join(effects, "state.json"), + JSON.stringify({ + scenario, + settlements: 0, + job, + assets: [ + { + id: "asset-0", + job_id: jobId, + kind: "quote_card", + ordinal: 0, + status: "pending", + attempt: 0, + provider_state: {}, + files: [], + caption: null, + qa: null, + error: null, + credits: 100, + maximum_usd: 0.01, + }, + ], + }), + ); + await writeFile(join(effects, "events.jsonl"), ""); + const ctx = await createFixtureWorld(temp, effects), + keepAlive = setInterval(() => {}, 1000); + try { + const run = await ctx.start(ctx.packSchedulerFixture, [ + { accountId: "fixture", packProtocol: 2 }, + jobId, + undefined, + { expected_revision: 1, execution_id: executionId, execution_version: 1 }, + ]); + let failure; + await waitFor(async () => { + try { + await run.returnValue; + } catch (error) { + failure = String(error); + return true; + } + return false; + }, 60000); + await waitFor(() => ctx.counts().activeDeliveries === 0); + const trace = (await readFile(join(effects, "events.jsonl"), "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map(JSON.parse); + assert(failure); + assert( + trace.some(x => x.action === "claim"), + "must reach asset claim", + ); + assert( + trace.some(x => x.action === "phase"), + "must reach provider phase", + ); + assert.match( + failure, + scenario === "phase-failure" + ? /Receipt acknowledgement lost/ + : /Settlement acknowledgement lost/, + ); + if (scenario === "settlement-failure") { + const finish = trace.findIndex(x => x.action === "finish_asset"); + assert(finish >= 0, "must finish asset before settlement loss"); + assert(finish < trace.findIndex(x => x.action === "settle")); + } + assert.equal( + trace.filter(x => x.action === "settle").length, + scenario === "phase-failure" ? 0 : 1, + ); + if (scenario === "phase-failure") + assert.equal(trace.filter(x => x.action === "finish_asset").length, 0); + const before = JSON.stringify(trace), + delivered = ctx.counts().deliveries; + await ctx.reenqueueRun(ctx.world, run.runId); + await waitFor(() => ctx.counts().deliveries > delivered); + const after = (await readFile(join(effects, "events.jsonl"), "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map(JSON.parse); + assert.equal(JSON.stringify(after), before); + const events = await ctx.world.events.list({ runId: run.runId }); + assert(events.data.some(x => x.eventType === "step_failed")); + results.push({ + scenario, + failure, + trace, + stepFailures: events.data.filter(x => x.eventType === "step_failed"), + terminalReplay: "no repeated effects", + }); + console.log( + `PASS ${scenario}: ${trace.filter(x => x.action === "settle").length} settlement call(s)`, + ); + } finally { + clearInterval(keepAlive); + await ctx.world.close(); + } + } + assert.equal(network(), 0); + assert.equal(additionalNetworkAttempts, 0); + if (process.env.PACK_FENCE_SDK_OUTPUT) + await writeFile( + process.env.PACK_FENCE_SDK_OUTPUT, + JSON.stringify( + { + bindings, + results, + networkAttempts: network() + additionalNetworkAttempts, + boundary: + "actual compiled release workflow body (export renamed only), actual steps; synthetic persistence/effect leaves, not SQL or hosted qualification", + }, + null, + 2, + ), + ); +} finally { + await rm(temp, { recursive: true, force: true }); +} diff --git a/scripts/pack-fence-workflow-fixture/source/createPackProductionServices.ts.txt b/scripts/pack-fence-workflow-fixture/source/createPackProductionServices.ts.txt new file mode 100644 index 000000000..6798e0298 --- /dev/null +++ b/scripts/pack-fence-workflow-fixture/source/createPackProductionServices.ts.txt @@ -0,0 +1,29 @@ +import { read } from "./read"; +import { record } from "./record"; +import { runPackCommand } from "./runPackCommand"; +import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError"; +import type { PackActor, PackJob, PackAsset } from "@/lib/content/packs/packTypes"; +import type { PackProductionServices } from "@/lib/content/packs/producePackAsset"; +export function createPackProductionServices( + actor: PackActor, + job: PackJob, + asset: PackAsset, +): PackProductionServices { + return { + phase: async () => { + record("phase"); + if (read().scenario === "phase-failure") + throw new PackFenceError("Receipt acknowledgement lost"); + }, + generateImages: async () => ["synthetic"], + generateMotion: async () => "synthetic", + caption: async () => "fixture", + compose: async () => ({ + files: [{ storage_key: "synthetic", content_type: "image/png", bytes: 1 }], + probe: {}, + }), + inspect: async () => ({ passed: true, reason: "fixture" }), + finish: result => + runPackCommand(actor, "finish_asset", job.id, { ...result, asset_id: asset.id }), + }; +} diff --git a/scripts/pack-fence-workflow-fixture/source/preparePackProduction.ts.txt b/scripts/pack-fence-workflow-fixture/source/preparePackProduction.ts.txt new file mode 100644 index 000000000..48bcca9c3 --- /dev/null +++ b/scripts/pack-fence-workflow-fixture/source/preparePackProduction.ts.txt @@ -0,0 +1,6 @@ +import { record } from "./record"; +import type { PackActor, PackJob } from "@/lib/content/packs/packTypes"; +export async function preparePackProduction(_actor: PackActor, job: PackJob) { + record("prepare"); + return job; +} diff --git a/scripts/pack-fence-workflow-fixture/source/read.ts.txt b/scripts/pack-fence-workflow-fixture/source/read.ts.txt new file mode 100644 index 000000000..3523a0d76 --- /dev/null +++ b/scripts/pack-fence-workflow-fixture/source/read.ts.txt @@ -0,0 +1,5 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +export function read() { + return JSON.parse(readFileSync(join(process.env.PACK_SCHEDULER_EFFECTS!, "state.json"), "utf8")); +} diff --git a/scripts/pack-fence-workflow-fixture/source/record.ts.txt b/scripts/pack-fence-workflow-fixture/source/record.ts.txt new file mode 100644 index 000000000..557591604 --- /dev/null +++ b/scripts/pack-fence-workflow-fixture/source/record.ts.txt @@ -0,0 +1,8 @@ +import { appendFileSync } from "node:fs"; +import { join } from "node:path"; +export function record(action: string) { + appendFileSync( + join(process.env.PACK_SCHEDULER_EFFECTS!, "events.jsonl"), + JSON.stringify({ action }) + "\n", + ); +} diff --git a/scripts/pack-fence-workflow-fixture/source/runPackCommand.ts.txt b/scripts/pack-fence-workflow-fixture/source/runPackCommand.ts.txt new file mode 100644 index 000000000..e7af034c5 --- /dev/null +++ b/scripts/pack-fence-workflow-fixture/source/runPackCommand.ts.txt @@ -0,0 +1,31 @@ +import { read } from "./read"; +import { write } from "./write"; +import { record } from "./record"; +import type { PackSnapshot } from "@/lib/content/packs/packTypes"; +export async function runPackCommand( + _actor: unknown, + action: string, + _job: string, + input: Record = {}, +): Promise { + const state = read(); + record(action); + if (action === "run") state.job.workflow_run_id = input.workflow_run_id; + if (action === "claim") { + const asset = state.assets.find(asset => asset.id === input.asset_id); + if (!asset || asset.status !== "pending") return { ...state, asset, claimed: false }; + asset.status = "running"; + asset.attempt += 1; + asset.claim_id = "00000000-0000-4000-8000-000000000003"; + write(state); + return { ...state, asset, claimed: true }; + } + if (action === "settle") { + state.settlements++; + write(state); + throw new Error("Settlement acknowledgement lost"); + } + if (action === "finish_asset") Object.assign(state.assets[0], input); + write(state); + return state; +} diff --git a/scripts/pack-fence-workflow-fixture/source/write.ts.txt b/scripts/pack-fence-workflow-fixture/source/write.ts.txt new file mode 100644 index 000000000..e88cdd4f9 --- /dev/null +++ b/scripts/pack-fence-workflow-fixture/source/write.ts.txt @@ -0,0 +1,5 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +export function write(state: unknown) { + writeFileSync(join(process.env.PACK_SCHEDULER_EFFECTS!, "state.json"), JSON.stringify(state)); +}