Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ jobs:
- name: Check actual internal fenced workflow with offline effect seams
run: pnpm test:chat-fence-workflow

- name: Check dormant pack worker fence with offline Workflow SDK
run: node scripts/pack-fence-workflow-fixture/run.mjs

- name: Check unwired pack scheduler replay with offline effects
run: pnpm test:pack-scheduler-workflow

Expand Down
41 changes: 36 additions & 5 deletions app/workflows/packs/foundation/createPackFoundationServices.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes";
import { assertPackExecutionIdentity } from "@/lib/content/packs/fence/assertPackExecutionIdentity";
import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError";
import { preparePackStep } from "../preparePackStep";
import { producePackAssetStep } from "../producePackAssetStep";
import { settleBoundedPackStep } from "../settleBoundedPackStep";
Expand All @@ -12,38 +15,66 @@ import type { PackFoundationServices } from "@/lib/content/packs/foundation/pack
export function createPackFoundationServices(
actor: PackActor,
jobId: string,
execution?: PackLaunchFence,
): PackFoundationServices {
if (actor.packProtocol === 2 && !execution)
throw new PackFenceError("Foundation requires original execution admission");
const attempts = new Map<string, number>();
const scope = (input: PackActor, id: string) => {
if (
id !== jobId ||
input.accountId !== actor.accountId ||
input.organizationId !== actor.organizationId
input.organizationId !== actor.organizationId ||
input.packProtocol !== actor.packProtocol
)
throw new Error("Foundation service scope mismatch");
};
return {
read: (a, id) => {
scope(a, id);
return readPackFoundationStep(a, id);
return readPackFoundationStep(a, id).then(saved => {
if (a.packProtocol === 2) {
assertPackExecutionIdentity(saved.job, execution);
for (const asset of saved.assets)
if (!attempts.has(asset.id)) attempts.set(asset.id, asset.attempt);
}
return saved;
});
},
prepare: (a, id) => {
scope(a, id);
return preparePackStep(a, id);
return preparePackStep(a, id, execution);
},
persist: (a, job, record) => {
scope(a, job.id);
if (a.packProtocol === 2) assertPackExecutionIdentity(job, execution);
return persistPackFoundationStep(a, job, record);
},
produce: (a, job, id) => {
scope(a, job.id);
if (a.packProtocol === 2) {
assertPackExecutionIdentity(job, execution);
return producePackAssetStep(a, job, id, false, attempts.get(id));
}
return producePackAssetStep(a, job, id);
},
settle: (a, id, error) => {
settle: (a, id, error, suppliedExecution) => {
scope(a, id);
return settleBoundedPackStep(a, id, error);
if (
a.packProtocol === 2 &&
(!suppliedExecution ||
suppliedExecution.execution_id !== execution!.execution_id ||
suppliedExecution.execution_version !== execution!.execution_version ||
suppliedExecution.expected_revision !== execution!.expected_revision)
)
throw new PackFenceError("Foundation settlement execution mismatch");
return a.packProtocol === 2
? settleBoundedPackStep(a, id, error, suppliedExecution)
: settleBoundedPackStep(a, id, error);
},
createPlan: (snapshot, selection) => {
scope(actor, snapshot.job.id);
if (actor.packProtocol === 2) assertPackExecutionIdentity(snapshot.job, execution);
return planPackFoundationStep(snapshot, selection);
},
bindFiles: (plan, asset) => bindPackFoundationFilesStep(actor, jobId, plan, asset),
Expand Down
27 changes: 24 additions & 3 deletions app/workflows/packs/preparePackStep.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,35 @@
import { assertPackExecutionIdentity } from "@/lib/content/packs/fence/assertPackExecutionIdentity";
import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes";
import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError";
import { getWorkflowMetadata } from "workflow";
import { runPackCommand } from "@/lib/supabase/release_pack_jobs/runPackCommand";
import { preparePackProduction } from "@/lib/content/packs/preparePackProduction";
import type { PackActor } from "@/lib/content/packs/packTypes";
export async function preparePackStep(actor: PackActor, jobId: string) {
export async function preparePackStep(
actor: PackActor,
jobId: string,
execution?: PackLaunchFence,
) {
"use step";
if (actor.packProtocol === 2 && !execution)
throw new PackFenceError("Missing original launch identity");
const workflowRunId = getWorkflowMetadata().workflowRunId;
await runPackCommand(actor, "run", jobId, {
workflow_run_id: getWorkflowMetadata().workflowRunId,
...execution,
workflow_run_id: workflowRunId,
});
const saved = await runPackCommand(actor, "read", jobId);
if (actor.packProtocol === 2) {
assertPackExecutionIdentity(saved.job, execution, workflowRunId);
if (saved.job.status !== "running") throw new PackFenceError("Pack no longer running");
}
const job = await preparePackProduction(actor, saved.job);
return { job, assetIds: saved.assets.filter(a => a.status === "pending").map(a => a.id) };
return {
job,
assetIds: saved.assets.filter(a => a.status === "pending").map(a => a.id),
...(actor.packProtocol === 2
? { assetAttempts: Object.fromEntries(saved.assets.map(a => [a.id, a.attempt] as const)) }
: {}),
};
}
preparePackStep.maxRetries = 0;
39 changes: 34 additions & 5 deletions app/workflows/packs/producePackAssetStep.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
import { getWorkflowMetadata } from "workflow";
import { getPackExecutionFence } from "@/lib/content/packs/fence/getPackExecutionFence";
import { getPackAssetFence } from "@/lib/content/packs/fence/getPackAssetFence";
import { getPackClaimRequestId } from "@/lib/content/packs/fence/getPackClaimRequestId";
import { isPackFenceError } from "@/lib/content/packs/fence/isPackFenceError";
import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError";
import { runPackCommand } from "@/lib/supabase/release_pack_jobs/runPackCommand";
import { producePackAsset } from "@/lib/content/packs/producePackAsset";
import { createPackProductionServices } from "@/lib/content/packs/createPackProductionServices";
Expand All @@ -7,22 +13,45 @@ export async function producePackAssetStep(
job: PackJob,
assetId: string,
resume = false,
expectedAttempt?: number,
) {
"use step";
const claim = resume
? await runPackCommand(actor, "read", job.id)
: await runPackCommand(actor, "claim", job.id, { asset_id: assetId });
const asset = resume ? claim.assets.find(a => a.id === assetId) : claim.asset;
if (!asset || (!resume && !claim.claimed)) return;
const fence = getPackExecutionFence(job);
if (actor.packProtocol === 2 && job.workflow_run_id !== getWorkflowMetadata().workflowRunId)
throw new PackFenceError("Actual workflow does not match original admission");
if ((actor.packProtocol === 2) !== (job.protocol === 2))
throw new PackFenceError("Pack protocol mismatch");
if (actor.packProtocol === 2 && !Number.isInteger(expectedAttempt))
throw new PackFenceError("Missing original pending attempt");
const claim =
resume && actor.packProtocol !== 2
? await runPackCommand(actor, "read", job.id)
: await runPackCommand(actor, "claim", job.id, {
...fence,
asset_id: assetId,
...(actor.packProtocol === 2
? {
asset_attempt: expectedAttempt,
request_id: getPackClaimRequestId(job.execution_id!, job.workflow_run_id!, assetId),
}
: {}),
});
if (actor.packProtocol === 2 && claim.replayed)
throw new PackFenceError("Claim replay is reconciliation, not another production permit");
const asset =
resume && actor.packProtocol !== 2 ? claim.assets.find(a => a.id === assetId) : claim.asset;
if (!asset || ((actor.packProtocol === 2 || !resume) && !claim.claimed)) return;
try {
await producePackAsset(job, asset, createPackProductionServices(actor, job, asset));
} catch (error) {
if (isPackFenceError(error)) throw error;
const message = error instanceof Error ? error.message : "Asset generation failed";
const uncertain = Object.entries(asset.provider_state).some(
([key, value]) =>
key.includes("submission") && (value as { state?: string }).state !== "complete",
);
await runPackCommand(actor, "finish_asset", job.id, {
...getPackAssetFence(job, asset),
asset_id: assetId,
status: uncertain ? "uncertain" : "failed",
files:
Expand Down
43 changes: 36 additions & 7 deletions app/workflows/packs/releasePackWorkflow.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes";
import { settleBoundedPackStep } from "./settleBoundedPackStep";
Comment thread
sidneyswift marked this conversation as resolved.
Resolved
import { preparePackStep } from "./preparePackStep";
import { producePackAssetStep } from "./producePackAssetStep";
import { settlePackStep } from "./settlePackStep";
Expand All @@ -7,17 +9,42 @@ export async function releasePackWorkflow(
actor: PackActor,
jobId: string,
resumedAssetId?: string,
execution?: PackLaunchFence,
) {
"use workflow";
try {
const prepared = await preparePackStep(actor, jobId);
if (resumedAssetId) await producePackAssetStep(actor, prepared.job, resumedAssetId, true);
else
for (const assetId of prepared.assetIds)
await producePackAssetStep(actor, prepared.job, assetId);
await settlePackStep(actor, jobId);
return { job_id: jobId };
const prepared = await preparePackStep(actor, jobId, execution);
if (resumedAssetId) {
if (actor.packProtocol === 2)
await producePackAssetStep(
actor,
prepared.job,
resumedAssetId,
true,
prepared.assetAttempts?.[resumedAssetId],
);
else await producePackAssetStep(actor, prepared.job, resumedAssetId, true);
} else
for (const assetId of prepared.assetIds) {
if (actor.packProtocol === 2)
await producePackAssetStep(
actor,
prepared.job,
assetId,
false,
prepared.assetAttempts?.[assetId],
);
else await producePackAssetStep(actor, prepared.job, assetId);
}
// Preserve the legacy workflow's existing settlement path. Protocol 2 does not retry an ambiguous settlement.
if (actor.packProtocol !== 2) {
await settlePackStep(actor, jobId);
return { job_id: jobId };
}
} catch (error) {
// Workflow SDK reconstructs terminal step errors as FatalError, losing custom names.
// Any protocol-2 step rejection therefore stops without fallback settlement.
if (actor.packProtocol === 2) throw error;
await settlePackStep(
actor,
jobId,
Expand All @@ -28,4 +55,6 @@ export async function releasePackWorkflow(
error: "Pack stopped; inspect saved status and outputs before retrying",
};
}
await settleBoundedPackStep(actor, jobId, undefined, execution);
return { job_id: jobId };
}
19 changes: 16 additions & 3 deletions app/workflows/packs/settleBoundedPackStep.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,21 @@
import { getWorkflowMetadata } from "workflow";
import type { PackLaunchFence } from "@/lib/content/packs/fence/packFenceTypes";
import { PackFenceError } from "@/lib/content/packs/fence/PackFenceError";
import { runPackCommand } from "@/lib/supabase/release_pack_jobs/runPackCommand";
import type { PackActor } from "@/lib/content/packs/packTypes";
/** Unwired scheduler adapter: an ambiguous settlement requires reconciliation, not retry. */
export async function settleBoundedPackStep(actor: PackActor, jobId: string, error?: string) {
/** Zero-retry adapter used by dormant protocol 2 and the unwired bounded scheduler: an ambiguous settlement requires reconciliation, not retry. */
export async function settleBoundedPackStep(
actor: PackActor,
jobId: string,
error?: string,
execution?: PackLaunchFence,
) {
"use step";
return runPackCommand(actor, "settle", jobId, { error: error ?? null });
if (actor.packProtocol === 2 && !execution)
throw new PackFenceError("Missing original settlement identity");
return runPackCommand(actor, "settle", jobId, {
error: error ?? null,
...(execution ? { ...execution, workflow_run_id: getWorkflowMetadata().workflowRunId } : {}),
});
}
settleBoundedPackStep.maxRetries = 0;
6 changes: 4 additions & 2 deletions lib/content/packs/BOUNDED_SCHEDULER.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ price, approval, reservation, or provider adapter is changed. There is no produc
concurrency default or environment switch.

The caller supplies a positive safe-integer concurrency bound and the existing
`preparePackStep`, `producePackAssetStep`, and the unwired `settleBoundedPackStep`
`preparePackStep`, `producePackAssetStep`, and `settleBoundedPackStep`
services. The settlement adapter uses the same command but sets `maxRetries = 0`;
the existing production `settlePackStep` and its retry behavior are unchanged.
the protocol-1 production `settlePackStep` and its retry behavior are unchanged. The zero-retry adapter is also wired into the dormant protocol-2 production workflow.
The controlled fixtures use explicit limits; they do not select a production cap.

## Execution contract
Expand Down Expand Up @@ -89,3 +89,5 @@ this helper with persisted dependency readiness for an already-quoted artwork as
It remains unwired, preserves independent eligibility, and reports explicit owner
continuations when prerequisites or reference capacity prevent dependent production.
Its source/unit qualification is separate from this helper's compiled replay evidence.

Protocol-2 exception: after draining in-flight work, any rejected preparation or production step stops without settlement. This conservative rule survives SDK error reconstruction and avoids treating an uncertain write as settlement authority. Successful runs settle once with the original execution fence; a settlement rejection is never retried here. The failure-settlement rules above describe protocol 1.
21 changes: 21 additions & 0 deletions lib/content/packs/__tests__/runBoundedPackProduction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -410,3 +410,24 @@ it("does not settle while final asset persistence is still in flight", async ()
expect(assets[0].status).toBe("completed");
expect(settlementActive).toEqual([0]);
});

it("never forwards an execution fence to legacy settlement", async () => {
const execution = {
expected_revision: 1,
execution_id: "00000000-0000-4000-8000-000000000001",
execution_version: 1,
};
const settle = vi.fn(async () => undefined);
await runBoundedPackProduction(
{ accountId: "owner" },
"job",
1,
{
prepare: async () => ({ job: {} as PackJob, assetIds: [] }),
produce: async () => undefined,
settle,
},
execution,
);
expect(settle).toHaveBeenCalledWith({ accountId: "owner" }, "job", undefined);
});
Loading
Loading