From 2877924e04227267dfb7709536b0329054b7a2aa Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:30:10 -0500 Subject: [PATCH 1/3] Propose inactive Pack execution fences and bounded receipt journal --- proposals/pack-worker-fence-2269/CONTRACT.md | 47 +++++ proposals/pack-worker-fence-2269/README.md | 27 +++ proposals/pack-worker-fence-2269/RECEIPTS.md | 46 +++++ proposals/pack-worker-fence-2269/command.sql | 186 ++++++++++++++++++ proposals/pack-worker-fence-2269/receipts.sql | 78 ++++++++ proposals/pack-worker-fence-2269/schema.sql | 76 +++++++ .../pack-worker-fence-2269/test_contract.py | 104 ++++++++++ .../pack-worker-fence-2269/test_extended.py | 108 ++++++++++ 8 files changed, 672 insertions(+) create mode 100644 proposals/pack-worker-fence-2269/CONTRACT.md create mode 100644 proposals/pack-worker-fence-2269/README.md create mode 100644 proposals/pack-worker-fence-2269/RECEIPTS.md create mode 100644 proposals/pack-worker-fence-2269/command.sql create mode 100644 proposals/pack-worker-fence-2269/receipts.sql create mode 100644 proposals/pack-worker-fence-2269/schema.sql create mode 100644 proposals/pack-worker-fence-2269/test_contract.py create mode 100644 proposals/pack-worker-fence-2269/test_extended.py diff --git a/proposals/pack-worker-fence-2269/CONTRACT.md b/proposals/pack-worker-fence-2269/CONTRACT.md new file mode 100644 index 0000000..1a48ac9 --- /dev/null +++ b/proposals/pack-worker-fence-2269/CONTRACT.md @@ -0,0 +1,47 @@ +# Paired execution and claim contract + +Implemented as inactive proposal SQL; see [README](README.md) for qualification and rollout boundaries. Current production/legacy commands remain unfenced. The preceding investigation found delayed phase, finish, context, run, settle, claim, dispatch and same-attempt-resume writes could mutate newer work. The proposal addresses those source-contract counterexamples; local tests are not a claim of hosted enforcement. + +## Identity and ownership + +Existing jobs default to protocol 1. A new protocol-2 prepare is explicit; an existing legacy job cannot be adopted. A guarded legacy wrapper rejects protocol-2 jobs, including idempotency-key resolution, and retains the original command for protocol 1. The original command is renamed to a private internal function; service-role direct Pack INSERT/UPDATE/DELETE and internal-function execution are revoked. Current API Pack persistence uses the RPC, not raw table DML. A future rollout must inventory other callers before applying those grants. + +Each newly accepted approval/resume creates an execution UUID and increasing job execution version. The transition request UUID and complete JSONB payload are retained. Source execution/version and creative revision are compare-and-set inputs. Resume also compares the original asset attempt/claim. Exact accepted transition replay returns its recorded result with `dispatch=false`; conflicting payload reuse refuses. Replay lookup precedes source-generation comparison, so an old accepted transition can be inspected without regaining write authority. An unseen stale resume cannot rotate a newer generation. + +The workflow receives the original accepted execution as an immutable launch argument. `run` binds one actual Workflow ID, null-to-ID or same-ID only; workflow IDs are unique across executions. Preparation checks the original identity again after reading the job. Production compares actual Workflow metadata before claiming. No worker repairs a mismatch using a current-row identity. + +A claim belongs to one execution and asset. New paid claims increment the existing attempt through the original command; uncertain resume preserves the paid attempt, billed credits and provider state but creates a new claim. Its immutable operation-ID array grants continuation of existing receipts. A deterministic claim request UUID is payload-bound. A repeated claim returns `claimed=false,replayed=true` and the API stops for reconciliation: an acknowledged old claim is not a second production permit. + +## Atomic write rules + +All commands recheck actor/org access; current organization membership is locked through the transaction. The lock order is job, then asset. Every worker validates revision, execution UUID/version and bound Workflow ID **before** expiry can settle/refund anything. Phase/finish additionally validate attempt/claim and retained execution association. Missing/null/wrong identities refuse as well as older valid identities. + +| Command | Additional rule | +| --- | --- | +| `run` | Compare-and-set Workflow ID; cannot adopt another run. | +| `dispatch_uncertain` | Original generation required; a bound run is not overwritten. | +| `claim` | Original expected pending attempt; one claim per execution/asset. | +| `phase` | Full claim fence; provider intent, request identity and transition order immutable; receipt append acknowledged before accepted/complete write. | +| `finish_asset` | Full claim fence, including catch writes; exact completion replay only. An incomplete provider submission cannot become a confirmed failure. | +| `context` | Original execution plus expected context version; acknowledged updates advance that version. Foundation read-modify-write preserves original execution. | +| `settle` | Original execution; retained exact terminal replay, one refund. Different payload refuses. | +| `prepare` update, `estimate`, `export` | Expected revision; retry estimate also compares expected failed attempt. | +| `read` | Authorized current-row recovery may expire current work; it is not an old worker write permit. | + +Stale refusal changes no Pack, wallet or usage state, including after the newer hold expires. A current admitted expiry settles once and records the terminal execution result. A subsequent worker call may receive a terminal refusal rather than a fresh permit; authorized read returns current state. Lost acknowledgements require reconciliation. + +Provider registration occurs atomically with `submitting`, before external I/O. A duplicate registration reports `operation_created=false`; the API never submits on that response. Accepted request charging remains inside the original command and occurs once. Completed state requires prior accepted state and its charge; changed request IDs, input/model/price, regression and conflicting completed payloads refuse. Existing non-provider phase values also cannot be overwritten with a different value. + +Allowed models and charges come from the quote's frozen execution configuration. Image slots are 0–3 for carousel, otherwise 0; motion is video-only. A retained ready foundation can select the quote's image-edit model for a dependent only when the supplied plan/asset digests match current durable context, job/owner/revision/quote match, the exact dependent attempt is bound, and the source artwork is completed. Original quote references otherwise determine image versus image-edit. No new rates are introduced. + +## API failure boundary + +Protocol 2 is an internal optional admission argument, never inferred from public request fields or environment flags. The existing public calls and legacy Workflow argument shapes remain unchanged. All normal and catch writes carry original fences. Receipt append has its own transaction and is acknowledged before an active phase write. A refused/uncertain write raises a fence error without provider resubmission or catch-path finish. + +The Workflow SDK reconstructs step errors, so the protocol-2 workflow stops on **any** rejected step rather than relying on a custom error class crossing that boundary. Settlement uses the existing zero-retry bounded step outside the production catch path. A lost settlement acknowledgement causes one attempt, no automatic retry. The legacy workflow retains its existing handling. Unwired bounded/foundation adapters propagate the same original admission and attempt identities; they are not activated by this change. + +## Evidence and limits + +Actual PostgreSQL checks cover stale writers before/after expiry, same-attempt resume, transition/claim/completion/settlement replay and conflict, concurrent claim/intent registration, context CAS, atomic charge refusal, receipt retention/access/bounds and the existing legacy controls. The paired API-to-PG fixture covers real claim/intent/journal/charge/completion, claim replay refusal, mismatched actual Workflow identity, foundation-dependent image-edit and a committed acceptance surviving an actual phase refusal without catch finish or charge. Compiled local SDK checks cover step failure, lost settlement acknowledgement and terminal replay, with networking denied. + +This does not cancel already-issued provider work, guarantee an external I/O fence stays current, attest provider responses, recover a response lost before durable recording, establish cache/QA freshness, or make legacy executions safe. Unknown intent cannot be automatically resubmitted. Operator reconciliation, rollout, hosted SQL, provider execution, paid runtime, renderer activation and customer outcomes remain outside this contribution. diff --git a/proposals/pack-worker-fence-2269/README.md b/proposals/pack-worker-fence-2269/README.md new file mode 100644 index 0000000..0f58037 --- /dev/null +++ b/proposals/pack-worker-fence-2269/README.md @@ -0,0 +1,27 @@ +# Inactive Pack worker-fence proposal + +Status: local paired implementation for review, 2026-11-18. **Do not apply to a hosted database.** These files are deliberately outside `supabase/migrations` and are not referenced by deployment runners. They do not change deployed behavior. + +The paired API branch is `codex/pack-worker-fence-2269` in `recoupable/api`. Its protocol-2 admission is internal and dormant; public REST/MCP callers continue to use protocol 1. Root owns paired acceptance, merge and any later rollout. Legacy workers do not gain a stale-worker guarantee from this proposal. + +## Files and local qualification + +- `schema.sql`: protocol ownership, execution/claim history, the two receipt tables and guarded legacy wrapper. +- `command.sql`: generation, workflow, claim and context fences; intent registration and existing-accounting composition. +- `receipts.sql`: separately committed, redacted observation append/read ports. +- `test_contract.py` and `test_extended.py`: actual command calls on disposable PostgreSQL, using the repository's existing synthetic fixture. +- `CONTRACT.md` and `RECEIPTS.md`: implemented behavior and remaining boundaries. + +Run with PostgreSQL 17 tools on PATH: + +```sh +PYTHONDONTWRITEBYTECODE=1 python3 proposals/pack-worker-fence-2269/test_extended.py +``` + +The fixture applies the existing Pack schema and then these three proposal files, in the order above, only to its owned temporary Unix-socket cluster. It uses no TCP listener or hosted credentials and cleans up its own cluster. The suite covers 11 protocol-2 cases and the existing 11 legacy controls. The paired API provides an additional runner that exercises real API production helpers against these exact SQL files; provider, media and caption effects are synthetic. Its compiled Workflow SDK fixture verifies failure/replay behavior separately. + +## Delivery and activation boundary + +Baseline: Database `1b892dd2671ba7e15e0c44b690fbfbe515b59fbd`; paired API `bb542a9924ba531e422b3cd229e75a8bdb1892da`. No historical migration is changed. Migration filename/ownership work in Database PRs #125/#135 and reservation composition in #134/#135 remain separate. Existing Pack accounting remains the implementation called internally; no new wallet, rates or reservation model is introduced. + +Before any rollout, a separately reviewed change must reconcile the current database head and migration ownership, convert this proposal into an ordered migration, verify deployed schema/permissions, inventory and drain or reconcile legacy in-flight executions, qualify the intended authenticated runtime, and explicitly admit new protocol-2 launches. This PR does none of those things. The renderer and foundation/scheduler candidates remain unwired. App 2269 remains open. diff --git a/proposals/pack-worker-fence-2269/RECEIPTS.md b/proposals/pack-worker-fence-2269/RECEIPTS.md new file mode 100644 index 0000000..1587910 --- /dev/null +++ b/proposals/pack-worker-fence-2269/RECEIPTS.md @@ -0,0 +1,46 @@ +# Pack receipt journal contract + +The inactive implementation has exactly two receipt tables, alongside the execution/claim history required by the fence. It is operational evidence, not a wallet, charge ledger or authority to spend. Existing mutable `provider_state`, charge events and Context lifecycle records cannot substitute for this history without losing original association or inventing unrelated identities. + +## Frozen operation + +`release_pack_provider_operations` is inserted in the successful fenced `submitting` transaction. It records database UUID/time, owner/job/asset, creative revision, original execution UUID/version, actual Workflow ID, claim/paid attempt, phase, provider (`fal`), model, input SHA-256 and quoted credit charge. Foreign keys retain parent/history rows; the command binds all owner and identity fields from its locked rows. No worker has direct table write permission. + +Unique `(owner_id, asset_id, attempt, phase)` permits one operation per paid phase. Registration returns `operation_created`; only a newly acknowledged intent permits submission. Resume keeps that operation and paid attempt and grants the new claim an immutable operation-ID continuation array. An original claim may append only to its original operation; a continuation may append only to its explicitly linked operations. Neither grants current job mutation. A fresh approved paid retry uses a new attempt. + +The command bounds phase slots: image 0–3 for carousel, otherwise image 0, and motion only for video; thus no more than five operation slots per paid attempt. Model/input hash/charge are frozen and validated against the quoted configuration. Evidence append cannot create attempts, operations, continuation grants, holds or charges. + +## Bounded observations + +`release_pack_provider_observations` has primary key `(operation_id, kind)`, where kind is accepted, completed, failed or first conflict; at most four rows per operation. Payloads are at most 1024 bytes. Accepted/completed/failed accept only: + +- `kind` and `request_id` (1–200 ASCII letters, digits, underscore or hyphen). +- Optional `output_sha256`, 64 lowercase hexadecimal characters, completed only. +- Required failed `reason_code`: `provider_failed`, `response_invalid` or `result_unavailable`. + +No raw results, content, prompts, URLs, headers, tokens, filenames, exception text or arbitrary extension fields. Unknown fields/type/size violations are rejected, never truncated. API builds this allowlist; SQL validates it independently. Output SHA is metadata, not proof of retained output or validated content. Provider identities are internal locators, not public progress fields. + +Acceptance precedes terminal observation. Exact payload replay returns `state=reused` and preserves the original timestamp. Changed payload in an occupied slot, differing request identity or conflicting success/failure preserves the original and appends at most one conflict row containing kind and canonical old/new payload hashes. Repeating that same conflict returns `conflict`; further distinct conflicts return `conflict_unretained_capacity`. No extra row or false retention claim is made. Invalid/unauthorized requests cannot consume that slot. + +## Authority and transactions + +Append/read ports run as dedicated `NOLOGIN pack_receipt_port`, with fixed `pg_catalog` search path and schema-qualified objects. It can SELECT operation/claim/job identity and SELECT/INSERT observations; it cannot update Pack, claim, wallet, hold or usage state. Narrow functions recheck actor/org membership and effective ownership. PUBLIC/browser execution is revoked, and service role gets function execution only. RLS is defense in depth, not a substitute for function authorization. The access helper locks current membership for the transaction. + +Append authorizes the original execution/claim or exact stored continuation, then serializes observations using a transaction-scoped operation advisory lock. It never invokes the Pack command or expiry recovery. A late original receipt therefore cannot mutate a newer hold. This trusted service boundary is not cryptographic provider attestation. + +Ordering: + +1. Fenced intent and operation commit before submit. Failed/uncertain/duplicate registration does not submit. +2. Accepted or completed observation commits in a separate RPC before its active phase write. +3. If that phase refuses or rolls back, the committed observation survives. The API stops without catch finish/settle or another provider submission. +4. An unacknowledged append is uncertain; exact replay/readback can establish retention, but this slice does not add an automatic reconciliation worker. A crash before first durable receipt remains a loss window. The original intent blocks automatic resubmission. + +Conflict responses commit their bounded row; invalid/unauthorized input raises without inserting. There is no append-then-raise stale check in the same transaction that would erase the evidence. + +The owner-scoped read port validates job and cursor ownership and returns at most 100 operation rows with nested observations, ordered by UUID. Pagination traverses the retained rows visible to each read, not a multi-call frozen snapshot; restart from the first page when reconciling newly registered operations. It is internal and unmounted, with no public receipt route. + +## Retention and qualification + +No TTL, pruning job, worker update/delete port or cascading parent deletion. Retain operations, observations and continuation history for the Pack lifetime, including after settlement/retry. Parent deletion is RESTRICT; any owner-data deletion requires a separately authorized administrative process. Growth is bounded per approved paid attempt, not falsely bounded per owner lifetime. Access revocation hides retained history without erasing it. + +Actual proposal tests register operations and continuation grants through the real fenced command, append via a separate service-role transaction, and then deliver an actual refused stale write. They check unchanged Pack/wallet/usage snapshots, bounded concurrent conflicts, owner/claim mismatch, pagination, redaction, privilege denial and same-attempt continuation. The paired API integration uses the same proposal files. These are disposable local PostgreSQL checks, not seeded-history-only mechanics, hosted qualification or live provider evidence. diff --git a/proposals/pack-worker-fence-2269/command.sql b/proposals/pack-worker-fence-2269/command.sql new file mode 100644 index 0000000..46345de --- /dev/null +++ b/proposals/pack-worker-fence-2269/command.sql @@ -0,0 +1,186 @@ +-- INACTIVE protocol-2 command; composes the unchanged pinned legacy accounting. +BEGIN; +CREATE FUNCTION public.release_pack_command_v2(p_actor uuid,p_org uuid,p_action text,p_job uuid,p_input jsonb DEFAULT '{}') +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$ +<> +DECLARE owner uuid; j public.release_pack_jobs; a public.release_pack_assets; + e public.release_pack_executions; c public.release_pack_claims; op public.release_pack_provider_operations; + result jsonb; value jsonb; previous jsonb; foundation jsonb; has_foundation boolean:=false; request uuid; fresh boolean:=false; + phase text; wanted_model text; wanted_credits bigint; continuation uuid[]; supplied jsonb:=p_input; +BEGIN + owner:=public.authorize_pack_actor(p_actor,p_org); + IF jsonb_typeof(p_input) IS DISTINCT FROM 'object' THEN RAISE EXCEPTION 'Invalid Pack input'; END IF; + IF p_action='prepare' AND NOT(p_input ? 'expected_revision') THEN + -- Lock new identity/idempotency resolution before changing protocol ownership. + PERFORM pg_advisory_xact_lock(hashtextextended(owner::text||':'||(p_input->>'idempotency_key'),0)); + IF EXISTS(SELECT 1 FROM public.release_pack_jobs WHERE owner_id=owner AND idempotency_key=p_input->>'idempotency_key' AND protocol<>2) THEN + RAISE EXCEPTION 'Legacy Pack cannot be adopted'; END IF; + result:=public.release_pack_command_legacy_internal(p_actor,p_org,p_action,p_job,p_input); + p_job:=(result->'job'->>'id')::uuid; + UPDATE public.release_pack_jobs SET protocol=2 WHERE id=p_job; + RETURN public.release_pack_command_legacy_internal(p_actor,p_org,'read',p_job,'{}'); + END IF; + SELECT * INTO STRICT j FROM public.release_pack_jobs WHERE id=p_job AND owner_id=owner FOR UPDATE; + IF j.protocol<>2 THEN RAISE EXCEPTION 'Legacy Pack cannot be adopted'; END IF; + IF p_action IN('approve','resume') THEN + request:=(p_input->>'request_id')::uuid; + IF request IS NULL THEN RAISE EXCEPTION 'Transition request identity required'; END IF; + SELECT * INTO e FROM public.release_pack_executions WHERE job_id=j.id AND transition_id=request; + IF FOUND THEN + IF e.transition_action<>p_action OR e.transition_payload<>p_input THEN RAISE EXCEPTION 'Transition payload conflict'; END IF; + RETURN e.transition_result||jsonb_build_object('dispatch',false); + END IF; + IF NOT(p_input ? 'source_execution_id') OR (p_input->>'source_execution_id')::uuid IS DISTINCT FROM j.execution_id + OR (p_input->>'source_execution_version')::integer IS DISTINCT FROM j.execution_version + OR (p_input->>'expected_revision')::integer IS DISTINCT FROM j.revision THEN RAISE EXCEPTION 'Stale transition'; END IF; + IF p_action='resume' THEN + SELECT * INTO STRICT a FROM public.release_pack_assets WHERE id=(p_input->>'asset_id')::uuid AND job_id=j.id FOR UPDATE; + IF (p_input->>'asset_attempt')::integer IS DISTINCT FROM a.attempt OR (p_input->>'source_claim_id')::uuid IS DISTINCT FROM a.claim_id OR a.claim_id IS NULL THEN RAISE EXCEPTION 'Stale resume claim'; END IF; + END IF; + result:=public.release_pack_command_legacy_internal(p_actor,p_org,p_action,j.id,p_input); + IF coalesce((result->>'command_expired')::boolean,false) OR NOT coalesce((result->>'dispatch')::boolean,false) THEN RETURN result; END IF; + INSERT INTO public.release_pack_executions(job_id,owner_id,revision,version,quote_id,transition_id,transition_action,transition_payload,resume_asset_id) + VALUES(j.id,owner,j.revision,j.execution_version+1,(j.quote->>'id')::uuid,request,p_action,p_input,CASE WHEN p_action='resume' THEN a.id END) RETURNING * INTO e; + UPDATE public.release_pack_jobs SET execution_id=e.id,execution_version=e.version WHERE id=j.id; + result:=public.release_pack_command_legacy_internal(p_actor,p_org,'read',j.id,'{}')||jsonb_build_object('dispatch',true); + UPDATE public.release_pack_executions SET transition_result=result WHERE id=e.id; + RETURN result; + ELSIF p_action IN('prepare','estimate','export') THEN + IF (p_input->>'expected_revision')::integer IS DISTINCT FROM j.revision THEN RAISE EXCEPTION 'Pack revision changed'; END IF; + IF p_action='estimate' AND p_input ? 'retry_asset_id' THEN + SELECT * INTO STRICT a FROM public.release_pack_assets WHERE id=(p_input->>'retry_asset_id')::uuid AND job_id=j.id FOR UPDATE; + IF (p_input->>'asset_attempt')::integer IS DISTINCT FROM a.attempt THEN RAISE EXCEPTION 'Retry attempt changed'; END IF; + END IF; + RETURN public.release_pack_command_legacy_internal(p_actor,p_org,p_action,j.id,p_input); + ELSIF p_action='read' THEN + result:=public.release_pack_command_legacy_internal(p_actor,p_org,'read',j.id,p_input); + IF j.status='running' AND result->'job'->>'status'<>'running' THEN + UPDATE public.release_pack_executions SET settlement_payload=jsonb_build_object('expiry',true),settlement_result=result WHERE id=j.execution_id AND settlement_result IS NULL; + END IF; + RETURN result; + END IF; + -- Every worker fence is checked BEFORE the legacy expiry/recovery branch can mutate. + IF (p_input->>'expected_revision')::integer IS DISTINCT FROM j.revision + OR (p_input->>'execution_id')::uuid IS DISTINCT FROM j.execution_id OR j.execution_id IS NULL + OR (p_input->>'execution_version')::integer IS DISTINCT FROM j.execution_version THEN RAISE EXCEPTION 'Stale Pack execution' USING ERRCODE='P0003'; END IF; + SELECT * INTO STRICT e FROM public.release_pack_executions WHERE id=j.execution_id AND owner_id=owner; + IF p_action='run' THEN + IF nullif(p_input->>'workflow_run_id','') IS NULL OR length(p_input->>'workflow_run_id')>200 THEN RAISE EXCEPTION 'Invalid workflow identity'; END IF; + IF e.workflow_run_id IS NOT NULL AND e.workflow_run_id<>p_input->>'workflow_run_id' THEN RAISE EXCEPTION 'Workflow identity conflict' USING ERRCODE='P0003'; END IF; + ELSIF p_action<>'dispatch_uncertain' THEN + IF e.workflow_run_id IS NULL OR p_input->>'workflow_run_id' IS DISTINCT FROM e.workflow_run_id THEN RAISE EXCEPTION 'Stale workflow' USING ERRCODE='P0003'; END IF; + END IF; + IF p_action IN('phase','finish_asset') THEN + SELECT * INTO STRICT a FROM public.release_pack_assets WHERE id=(p_input->>'asset_id')::uuid AND job_id=j.id FOR UPDATE; + IF (p_input->>'asset_attempt')::integer IS DISTINCT FROM a.attempt OR (p_input->>'claim_id')::uuid IS DISTINCT FROM a.claim_id OR a.claim_id IS NULL THEN RAISE EXCEPTION 'Stale asset claim' USING ERRCODE='P0003'; END IF; + SELECT * INTO STRICT c FROM public.release_pack_claims WHERE id=a.claim_id AND execution_id=e.id AND owner_id=owner; + IF p_action='finish_asset' AND c.completion_result IS NOT NULL THEN + IF c.completion_payload<>p_input THEN RAISE EXCEPTION 'Completion payload conflict'; END IF; + RETURN c.completion_result; + END IF; + IF a.status<>'running' THEN RAISE EXCEPTION 'Asset not running'; END IF; + END IF; + IF p_action='settle' AND e.settlement_result IS NOT NULL THEN + IF e.settlement_payload<>p_input THEN RAISE EXCEPTION 'Settlement payload conflict'; END IF; + RETURN e.settlement_result; + END IF; + IF j.status<>'running' THEN RAISE EXCEPTION 'Pack not running'; END IF; + IF j.hold_expires_at>'workflow_run_id' WHERE id=e.id AND workflow_run_id IS NULL; + ELSIF p_action='dispatch_uncertain' THEN + IF e.workflow_run_id IS NOT NULL THEN RETURN public.release_pack_command_legacy_internal(p_actor,p_org,'read',j.id,'{}'); END IF; + ELSIF p_action='claim' THEN + request:=(p_input->>'request_id')::uuid; + IF request IS NULL THEN RAISE EXCEPTION 'Claim request identity required'; END IF; + SELECT * INTO STRICT a FROM public.release_pack_assets WHERE id=(p_input->>'asset_id')::uuid AND job_id=j.id FOR UPDATE; + SELECT * INTO c FROM public.release_pack_claims WHERE execution_id=e.id AND asset_id=a.id; + IF FOUND THEN + IF c.request_id=request THEN + IF c.request_payload<>p_input THEN RAISE EXCEPTION 'Claim payload conflict'; END IF; + RETURN jsonb_build_object('job',to_jsonb(j),'asset',to_jsonb(a),'claimed',false,'replayed',true); + END IF; + RETURN jsonb_build_object('job',to_jsonb(j),'asset',to_jsonb(a),'claimed',false); + END IF; + IF (p_input->>'asset_attempt')::integer IS DISTINCT FROM a.attempt THEN RAISE EXCEPTION 'Stale pending attempt' USING ERRCODE='P0003'; END IF; + IF e.resume_asset_id=a.id THEN + IF a.status<>'running' THEN RAISE EXCEPTION 'Resume asset not running'; END IF; + SELECT coalesce(array_agg(id),'{}') INTO continuation FROM public.release_pack_provider_operations WHERE owner_id=owner AND asset_id=a.id AND attempt=a.attempt; + ELSE + result:=public.release_pack_command_legacy_internal(p_actor,p_org,'claim',j.id,p_input); + IF NOT coalesce((result->>'claimed')::boolean,false) THEN RETURN result; END IF; + SELECT * INTO STRICT a FROM public.release_pack_assets WHERE id=a.id; + continuation:='{}'; + END IF; + INSERT INTO public.release_pack_claims(execution_id,owner_id,asset_id,attempt,request_id,request_payload,operation_ids) + VALUES(e.id,owner,a.id,a.attempt,request,p_input,continuation) RETURNING * INTO c; + UPDATE public.release_pack_assets SET claim_id=c.id WHERE id=a.id RETURNING * INTO a; + result:=jsonb_build_object('job',to_jsonb(j),'asset',to_jsonb(a),'claimed',true); + UPDATE public.release_pack_claims SET claim_result=result WHERE id=c.id; + RETURN result; + ELSIF p_action='context' THEN + IF (p_input->>'context_version')::integer IS DISTINCT FROM j.context_version THEN RAISE EXCEPTION 'Context version changed'; END IF; + UPDATE public.release_pack_jobs SET context_version=context_version+1 WHERE id=j.id; + ELSIF p_action='phase' THEN + phase:=p_input->>'phase'; value:=p_input->'value'; previous:=a.provider_state->phase; + IF phase IS NULL OR length(phase)>100 OR value IS NULL THEN RAISE EXCEPTION 'Invalid phase'; END IF; + IF phase LIKE '%submission%' THEN + IF p_input ? 'shared_foundation_binding' THEN + foundation:=j.production_context->'shared_foundation'; + IF foundation->>'state' IS DISTINCT FROM 'ready' + OR p_input->'shared_foundation_binding' IS DISTINCT FROM jsonb_build_object('plan_sha256',foundation->'plan'->>'digest','asset_sha256',foundation->>'asset_sha256') + OR foundation->'plan'->>'job_id' IS DISTINCT FROM j.id::text + OR foundation->'plan'->>'owner_id' IS DISTINCT FROM owner::text + OR (foundation->'plan'->>'revision')::integer IS DISTINCT FROM j.revision + OR foundation->'plan'->>'quote_id' IS DISTINCT FROM j.quote->>'id' + OR NOT coalesce(foundation->'plan'->'dependent_asset_ids' ? a.id::text,false) + OR NOT coalesce(foundation->'dependency_bindings' @> jsonb_build_array(jsonb_build_object('asset_id',a.id,'attempt',a.attempt)),false) + OR NOT EXISTS(SELECT 1 FROM public.release_pack_assets source WHERE source.id=(foundation->'plan'->>'asset_id')::uuid AND source.job_id=j.id AND source.kind='artwork' AND source.status='completed') + THEN RAISE EXCEPTION 'Retained foundation binding changed'; END IF; + has_foundation:=true; + END IF; + IF phase='motion_submission' AND a.kind='video' THEN + wanted_model:=j.quote->'execution_config'->'models'->>'motion'; + wanted_credits:=greatest(1,round((j.deliverables->>'video_seconds')::numeric*(j.quote->'execution_config'->'rates'->>'video_units_per_second')::numeric*(j.quote->'execution_config'->'rates'->>'video_usd_per_unit')::numeric*(j.quote->'execution_config'->'rates'->>'credits_per_usd')::numeric)); + ELSIF phase ~ '^image_submission_[0-3]$' AND (a.kind='carousel' OR phase='image_submission_0') THEN + wanted_model:=CASE WHEN has_foundation OR coalesce(jsonb_array_length(j.quote->'sources'->'references'),jsonb_array_length(j.brief->'reference_image_urls'),0)>0 THEN j.quote->'execution_config'->'models'->>'image_edit' ELSE j.quote->'execution_config'->'models'->>'image' END; + wanted_credits:=greatest(1,round((j.quote->'execution_config'->'rates'->>'image_usd_per_unit')::numeric*(j.quote->'execution_config'->'rates'->>'credits_per_usd')::numeric)); + ELSE RAISE EXCEPTION 'Provider phase outside quoted plan'; END IF; + SELECT o.* INTO op FROM public.release_pack_provider_operations o WHERE o.owner_id=owner AND o.asset_id=a.id AND o.attempt=a.attempt AND o.phase=cmd.phase; + IF NOT FOUND THEN + IF value->>'state' IS DISTINCT FROM 'submitting' OR wanted_model IS NULL OR value->>'model' IS DISTINCT FROM wanted_model + OR coalesce(value->>'input_sha256','') !~ '^[a-f0-9]{64}$' OR (value->>'charge_credits')::bigint IS NULL + OR wanted_credits IS NULL OR (value->>'charge_credits')::bigint IS DISTINCT FROM wanted_credits + OR (value->>'charge_credits')::bigint<=0 OR (value->>'charge_credits')::bigint>a.credits-a.billed_credits THEN RAISE EXCEPTION 'Invalid quoted provider intent'; END IF; + fresh:=true; + INSERT INTO public.release_pack_provider_operations(owner_id,job_id,asset_id,revision,execution_id,execution_version,workflow_run_id,claim_id,attempt,phase,provider,model,input_sha256,charge_credits) + VALUES(owner,j.id,a.id,j.revision,e.id,e.version,e.workflow_run_id,c.id,a.attempt,phase,'fal',wanted_model,value->>'input_sha256',(value->>'charge_credits')::bigint) RETURNING * INTO op; + END IF; + IF value->>'model' IS DISTINCT FROM op.model OR value->>'input_sha256' IS DISTINCT FROM op.input_sha256 OR (value->>'charge_credits')::bigint IS DISTINCT FROM op.charge_credits THEN RAISE EXCEPTION 'Provider intent identity changed'; END IF; + IF coalesce(value->>'state','') NOT IN('submitting','accepted','complete') THEN RAISE EXCEPTION 'Invalid provider state'; END IF; + IF value->>'state' IN('accepted','complete') AND NOT EXISTS(SELECT 1 FROM public.release_pack_provider_observations o WHERE operation_id=op.id AND kind=CASE WHEN value->>'state'='accepted' THEN 'accepted' ELSE 'completed' END AND payload->>'request_id'=value->>'request_id') THEN RAISE EXCEPTION 'Durable observation required before phase'; END IF; + IF previous->>'request_id' IS NOT NULL AND (value->>'request_id' IS DISTINCT FROM previous->>'request_id' OR value->>'state'='submitting') THEN RAISE EXCEPTION 'Provider receipt conflict'; END IF; + IF value->>'state'='complete' AND coalesce(previous->>'state','') NOT IN('accepted','complete') THEN RAISE EXCEPTION 'Accepted charge must precede completion'; END IF; + value:=value||jsonb_build_object('operation_id',op.id); + IF previous->>'state'='complete' AND previous<>value THEN RAISE EXCEPTION 'Completed receipt is immutable'; END IF; + p_input:=p_input||jsonb_build_object('value',value); + ELSIF previous IS NOT NULL AND previous<>value THEN RAISE EXCEPTION 'Saved phase conflict'; END IF; + ELSIF p_action='finish_asset' THEN + IF p_input->>'status'='failed' AND EXISTS(SELECT 1 FROM jsonb_each(a.provider_state) entry WHERE entry.key LIKE '%submission%' AND entry.value->>'state' IS DISTINCT FROM 'complete') THEN + RAISE EXCEPTION 'Incomplete provider attempt requires uncertainty'; + END IF; + ELSIF p_action<>'settle' THEN RAISE EXCEPTION 'Unknown protocol-2 command'; + END IF; + result:=public.release_pack_command_legacy_internal(p_actor,p_org,p_action,j.id,p_input); + IF p_action='finish_asset' THEN UPDATE public.release_pack_claims SET completion_payload=supplied,completion_result=result WHERE id=c.id; END IF; + IF p_action='settle' THEN UPDATE public.release_pack_executions SET settlement_payload=supplied,settlement_result=result WHERE id=e.id; END IF; + IF op.id IS NOT NULL THEN result:=result||jsonb_build_object('operation_id',op.id,'operation_created',fresh); END IF; + RETURN result; +END $$; +REVOKE ALL ON FUNCTION public.release_pack_command_v2(uuid,uuid,text,uuid,jsonb) FROM PUBLIC,anon,authenticated; +GRANT EXECUTE ON FUNCTION public.release_pack_command_v2(uuid,uuid,text,uuid,jsonb) TO service_role; +COMMIT; diff --git a/proposals/pack-worker-fence-2269/receipts.sql b/proposals/pack-worker-fence-2269/receipts.sql new file mode 100644 index 0000000..072afb0 --- /dev/null +++ b/proposals/pack-worker-fence-2269/receipts.sql @@ -0,0 +1,78 @@ +-- INACTIVE journal ports. Dedicated owner cannot mutate Pack/accounting rows. +BEGIN; +CREATE ROLE pack_receipt_port NOLOGIN; +GRANT USAGE ON SCHEMA public TO pack_receipt_port; +GRANT EXECUTE ON FUNCTION public.authorize_pack_actor(uuid,uuid) TO pack_receipt_port; +GRANT SELECT ON public.release_pack_provider_operations,public.release_pack_claims,public.release_pack_jobs TO pack_receipt_port; +GRANT SELECT,INSERT ON public.release_pack_provider_observations TO pack_receipt_port; +CREATE POLICY receipt_operation_read ON public.release_pack_provider_operations FOR SELECT TO pack_receipt_port USING(true); +CREATE POLICY receipt_claim_read ON public.release_pack_claims FOR SELECT TO pack_receipt_port USING(true); +CREATE POLICY receipt_job_read ON public.release_pack_jobs FOR SELECT TO pack_receipt_port USING(true); +CREATE POLICY receipt_observation_read ON public.release_pack_provider_observations FOR SELECT TO pack_receipt_port USING(true); +CREATE POLICY receipt_observation_insert ON public.release_pack_provider_observations FOR INSERT TO pack_receipt_port WITH CHECK(true); +CREATE FUNCTION public.append_pack_provider_observation(p_actor uuid,p_org uuid,p_operation uuid, + p_identity jsonb,p_payload jsonb) RETURNS jsonb +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$ +DECLARE op public.release_pack_provider_operations; effective_owner uuid; k text; + existing jsonb; accepted jsonb; terminal jsonb; conflict jsonb; previous_conflict jsonb; +BEGIN + effective_owner:=public.authorize_pack_actor(p_actor,p_org); + SELECT * INTO STRICT op FROM public.release_pack_provider_operations WHERE id=p_operation AND owner_id=effective_owner; + IF NOT (((p_identity->>'execution_id')::uuid=op.execution_id AND (p_identity->>'claim_id')::uuid=op.claim_id) OR EXISTS( + SELECT 1 FROM public.release_pack_claims WHERE owner_id=effective_owner AND op.id=ANY(operation_ids) + AND execution_id=(p_identity->>'execution_id')::uuid AND id=(p_identity->>'claim_id')::uuid + )) OR (p_identity->>'execution_id')::uuid IS NULL OR (p_identity->>'claim_id')::uuid IS NULL THEN RAISE EXCEPTION 'Receipt identity denied'; END IF; + IF jsonb_typeof(p_payload) IS DISTINCT FROM 'object' OR octet_length(p_payload::text)>1024 + OR p_payload-ARRAY['kind','request_id','output_sha256','reason_code']<>'{}'::jsonb + OR jsonb_typeof(p_payload->'kind') IS DISTINCT FROM 'string' + OR jsonb_typeof(p_payload->'request_id') IS DISTINCT FROM 'string' + OR coalesce(p_payload->>'request_id','') !~ '^[A-Za-z0-9_-]{1,200}$' + THEN RAISE EXCEPTION 'Invalid redacted observation'; END IF; + k:=p_payload->>'kind'; + IF k NOT IN ('accepted','completed','failed') + OR (p_payload ? 'output_sha256' AND (k<>'completed' OR jsonb_typeof(p_payload->'output_sha256') IS DISTINCT FROM 'string' OR p_payload->>'output_sha256' !~ '^[a-f0-9]{64}$')) + OR (k='failed' AND coalesce(p_payload->>'reason_code','') NOT IN ('provider_failed','response_invalid','result_unavailable')) + OR (k<>'failed' AND p_payload ? 'reason_code') THEN RAISE EXCEPTION 'Invalid observation state'; END IF; + -- Advisory lock avoids granting UPDATE on immutable operation rows. + PERFORM pg_advisory_xact_lock(hashtextextended(op.id::text,0)); + SELECT payload INTO existing FROM public.release_pack_provider_observations WHERE operation_id=op.id AND kind=k; + IF existing=p_payload THEN RETURN jsonb_build_object('state','reused','kind',k); END IF; + SELECT payload INTO accepted FROM public.release_pack_provider_observations WHERE operation_id=op.id AND kind='accepted'; + IF k<>'accepted' AND accepted IS NULL THEN RAISE EXCEPTION 'Acceptance must be durable first'; END IF; + SELECT payload INTO terminal FROM public.release_pack_provider_observations WHERE operation_id=op.id + AND kind IN ('completed','failed') AND kind<>k LIMIT 1; + IF existing IS NOT NULL OR (accepted IS NOT NULL AND accepted->>'request_id'<>p_payload->>'request_id') OR (k<>'accepted' AND terminal IS NOT NULL) THEN + conflict:=jsonb_build_object('kind',k,'existing_sha256',encode(sha256(convert_to(coalesce(existing,terminal,accepted)::text,'UTF8')),'hex'), + 'incoming_sha256',encode(sha256(convert_to(p_payload::text,'UTF8')),'hex')); + SELECT payload INTO previous_conflict FROM public.release_pack_provider_observations WHERE operation_id=op.id AND kind='conflict'; + IF previous_conflict IS NULL THEN + INSERT INTO public.release_pack_provider_observations VALUES(op.id,'conflict',conflict,clock_timestamp()); + RETURN jsonb_build_object('state','conflict','retained',true); + END IF; + RETURN jsonb_build_object('state',CASE WHEN previous_conflict=conflict THEN 'conflict' ELSE 'conflict_unretained_capacity' END,'retained',previous_conflict=conflict); + END IF; + INSERT INTO public.release_pack_provider_observations VALUES(op.id,k,p_payload,clock_timestamp()); + RETURN jsonb_build_object('state','saved','kind',k); +END $$; + +CREATE FUNCTION public.read_pack_provider_observations(p_actor uuid,p_org uuid,p_job uuid,p_after uuid DEFAULT NULL,p_limit integer DEFAULT 100) +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$ +DECLARE effective_owner uuid; rows jsonb; +BEGIN + effective_owner:=public.authorize_pack_actor(p_actor,p_org); + IF p_limit IS NULL OR p_limit NOT BETWEEN 1 AND 100 THEN RAISE EXCEPTION 'Invalid receipt page'; END IF; + IF NOT EXISTS(SELECT 1 FROM public.release_pack_jobs WHERE id=p_job AND owner_id=effective_owner) THEN RAISE EXCEPTION 'Pack access denied'; END IF; + IF p_after IS NOT NULL AND NOT EXISTS(SELECT 1 FROM public.release_pack_provider_operations WHERE id=p_after AND job_id=p_job AND owner_id=effective_owner) THEN RAISE EXCEPTION 'Receipt cursor denied'; END IF; + SELECT coalesce(jsonb_agg(value ORDER BY id),'[]') INTO rows FROM ( + SELECT op.id,to_jsonb(op)||jsonb_build_object('observations',(SELECT coalesce(jsonb_agg(to_jsonb(o) ORDER BY kind),'[]') FROM public.release_pack_provider_observations o WHERE o.operation_id=op.id)) AS value + FROM public.release_pack_provider_operations op WHERE op.owner_id=effective_owner AND op.job_id=p_job AND (p_after IS NULL OR op.id>p_after) ORDER BY op.id LIMIT p_limit + ) page; + RETURN jsonb_build_object('operations',rows,'next_cursor',CASE WHEN jsonb_array_length(rows)=p_limit AND EXISTS(SELECT 1 FROM public.release_pack_provider_operations WHERE job_id=p_job AND owner_id=effective_owner AND id>(rows->-1->>'id')::uuid) THEN rows->-1->>'id' END); +END $$; +GRANT CREATE ON SCHEMA public TO pack_receipt_port; +ALTER FUNCTION public.append_pack_provider_observation(uuid,uuid,uuid,jsonb,jsonb) OWNER TO pack_receipt_port; +ALTER FUNCTION public.read_pack_provider_observations(uuid,uuid,uuid,uuid,integer) OWNER TO pack_receipt_port; +REVOKE CREATE ON SCHEMA public FROM pack_receipt_port; +REVOKE ALL ON FUNCTION public.append_pack_provider_observation(uuid,uuid,uuid,jsonb,jsonb),public.read_pack_provider_observations(uuid,uuid,uuid,uuid,integer) FROM PUBLIC,anon,authenticated; +GRANT EXECUTE ON FUNCTION public.append_pack_provider_observation(uuid,uuid,uuid,jsonb,jsonb),public.read_pack_provider_observations(uuid,uuid,uuid,uuid,integer) TO service_role; +COMMIT; diff --git a/proposals/pack-worker-fence-2269/schema.sql b/proposals/pack-worker-fence-2269/schema.sql new file mode 100644 index 0000000..169d605 --- /dev/null +++ b/proposals/pack-worker-fence-2269/schema.sql @@ -0,0 +1,76 @@ +-- INACTIVE proposal. Apply only to disposable fixtures; never a migration runner. +BEGIN; +ALTER TABLE public.release_pack_jobs ADD COLUMN protocol integer NOT NULL DEFAULT 1 CHECK(protocol IN(1,2)), + ADD COLUMN execution_id uuid, ADD COLUMN execution_version integer NOT NULL DEFAULT 0, + ADD COLUMN context_version integer NOT NULL DEFAULT 0; +ALTER TABLE public.release_pack_assets ADD COLUMN claim_id uuid; +CREATE TABLE public.release_pack_executions ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), job_id uuid NOT NULL REFERENCES public.release_pack_jobs(id) ON DELETE RESTRICT, + owner_id uuid NOT NULL, revision integer NOT NULL, version integer NOT NULL, + quote_id uuid NOT NULL, transition_id uuid NOT NULL, transition_action text NOT NULL, + transition_payload jsonb NOT NULL, transition_result jsonb, workflow_run_id text UNIQUE, + resume_asset_id uuid, settlement_payload jsonb, settlement_result jsonb, + created_at timestamptz NOT NULL DEFAULT now(), UNIQUE(job_id,version), UNIQUE(job_id,transition_id) +); +CREATE TABLE public.release_pack_claims ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), execution_id uuid NOT NULL REFERENCES public.release_pack_executions(id) ON DELETE RESTRICT, + owner_id uuid NOT NULL, asset_id uuid NOT NULL REFERENCES public.release_pack_assets(id) ON DELETE RESTRICT, + attempt integer NOT NULL, request_id uuid NOT NULL, request_payload jsonb NOT NULL, + claim_result jsonb, completion_payload jsonb, completion_result jsonb, + operation_ids uuid[] NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE(execution_id,asset_id), CHECK(cardinality(operation_ids)<=5) +); +CREATE TABLE public.release_pack_provider_operations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), owner_id uuid NOT NULL, + job_id uuid NOT NULL REFERENCES public.release_pack_jobs(id) ON DELETE RESTRICT, + asset_id uuid NOT NULL REFERENCES public.release_pack_assets(id) ON DELETE RESTRICT, + revision integer NOT NULL, execution_id uuid NOT NULL REFERENCES public.release_pack_executions(id) ON DELETE RESTRICT, + execution_version integer NOT NULL, workflow_run_id text NOT NULL, + claim_id uuid NOT NULL REFERENCES public.release_pack_claims(id) ON DELETE RESTRICT, + attempt integer NOT NULL, phase text NOT NULL, provider text NOT NULL CHECK(provider='fal'), + model text NOT NULL CHECK(length(model) BETWEEN 1 AND 200), + input_sha256 text NOT NULL CHECK(input_sha256 ~ '^[a-f0-9]{64}$'), + charge_credits bigint NOT NULL CHECK(charge_credits>0), created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE(owner_id,asset_id,attempt,phase) +); +CREATE TABLE public.release_pack_provider_observations ( + operation_id uuid NOT NULL REFERENCES public.release_pack_provider_operations(id) ON DELETE RESTRICT, + kind text NOT NULL CHECK(kind IN('accepted','completed','failed','conflict')), + payload jsonb NOT NULL CHECK(octet_length(payload::text)<=1024), + recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(), PRIMARY KEY(operation_id,kind) +); +ALTER TABLE public.release_pack_executions ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.release_pack_claims ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.release_pack_provider_operations ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.release_pack_provider_observations ENABLE ROW LEVEL SECURITY; +REVOKE ALL ON public.release_pack_executions,public.release_pack_claims,public.release_pack_provider_operations,public.release_pack_provider_observations FROM PUBLIC,anon,authenticated,service_role; +-- Existing callers use the RPC. No raw worker DML may bypass protocol ownership. +REVOKE INSERT,UPDATE,DELETE ON public.release_pack_jobs,public.release_pack_assets FROM service_role; +ALTER FUNCTION public.release_pack_command(uuid,uuid,text,uuid,jsonb) RENAME TO release_pack_command_legacy_internal; +REVOKE ALL ON FUNCTION public.release_pack_command_legacy_internal(uuid,uuid,text,uuid,jsonb) FROM PUBLIC,anon,authenticated,service_role; +CREATE FUNCTION public.authorize_pack_actor(p_actor uuid,p_org uuid) RETURNS uuid +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$ +BEGIN + IF p_actor IS NULL THEN RAISE EXCEPTION 'Pack access denied' USING ERRCODE='42501'; END IF; + IF p_org IS NOT NULL AND p_org<>p_actor THEN + PERFORM 1 FROM public.account_organization_ids WHERE account_id=p_actor AND organization_id=p_org FOR SHARE; + IF NOT FOUND THEN RAISE EXCEPTION 'Pack access denied' USING ERRCODE='42501'; END IF; + END IF; + RETURN coalesce(p_org,p_actor); +END $$; +REVOKE ALL ON FUNCTION public.authorize_pack_actor(uuid,uuid) FROM PUBLIC,anon,authenticated,service_role; +CREATE FUNCTION public.release_pack_command(p_actor uuid,p_org uuid,p_action text,p_job uuid,p_input jsonb DEFAULT '{}') +RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$ +DECLARE result jsonb; +BEGIN + PERFORM public.authorize_pack_actor(p_actor,p_org); + IF EXISTS(SELECT 1 FROM public.release_pack_jobs WHERE id=p_job AND protocol=2) THEN + RAISE EXCEPTION 'Protocol 2 requires a fenced caller'; END IF; + result:=public.release_pack_command_legacy_internal(p_actor,p_org,p_action,p_job,p_input); + -- Also protects idempotency-key resolution and a concurrent prepare at another ID. + IF (result->'job'->>'protocol')::integer=2 THEN RAISE EXCEPTION 'Protocol 2 requires a fenced caller'; END IF; + RETURN result; +END $$; +REVOKE ALL ON FUNCTION public.release_pack_command(uuid,uuid,text,uuid,jsonb) FROM PUBLIC,anon,authenticated; +GRANT EXECUTE ON FUNCTION public.release_pack_command(uuid,uuid,text,uuid,jsonb) TO service_role; +COMMIT; diff --git a/proposals/pack-worker-fence-2269/test_contract.py b/proposals/pack-worker-fence-2269/test_contract.py new file mode 100644 index 0000000..6a0f250 --- /dev/null +++ b/proposals/pack-worker-fence-2269/test_contract.py @@ -0,0 +1,104 @@ +"""Actual inactive proposal on disposable PostgreSQL. No hosted credentials.""" +import json +import pathlib +import sys +import unittest +import uuid +import concurrent.futures +ROOT = pathlib.Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +from test_release_pack_jobs import ReleasePackJobs + +class FencedPacks(ReleasePackJobs): + @classmethod + def setUpClass(cls): + super().setUpClass() + for name in ['schema.sql','command.sql','receipts.sql']: + path=pathlib.Path(__file__).parent/name + if path.exists(): cls.sql(path.read_text()) + def command(self, action, payload=None, actor=None, org=None, succeeds=True, legacy=False): + data=json.dumps(payload or {}).replace("'","''") + orgsql="'"+org+"'" if org else 'null' + name='release_pack_command' if legacy else 'release_pack_command_v2' + result=self.sql(f"SET ROLE service_role; SELECT {name}('{actor or self.owner}',{orgsql},'{action}','{self.job}','{data}'::jsonb)",succeeds=succeeds) + return json.loads(result.stdout) if succeeds else result + def setup_quote(self): + self.prepare() + q={'id':str(uuid.uuid4()),'revision':1,'credits':100,'maximum_usd':.01,'expires_at':'2099-01-01T00:00:00Z', + 'execution_config':{'models':{'image':'fixture','image_edit':'fixture-edit','motion':'fixture-motion'},'rates':{'image_usd_per_unit':.004,'credits_per_usd':10000}}} + return self.command('estimate',{'expected_revision':1,'quote':q,'assets':[{'kind':getattr(self,'asset_kind','artwork'),'ordinal':0,'credits':100,'maximum_usd':.01}]}) + def approve_v2(self): + saved=self.setup_quote();q=saved['job']['quote'] + payload={'request_id':str(uuid.uuid4()),'quote_id':q['id'],'expected_revision':1,'approved_max_usd':.01,'source_execution_id':None,'source_execution_version':0} + funded=self.command('approve',payload) + return funded,payload + def bind(self,saved,run='run-1'): + j=saved['job']; run=self.job+'-'+run; f={'expected_revision':j['revision'],'execution_id':j['execution_id'],'execution_version':j['execution_version'],'workflow_run_id':run} + self.command('run',f) + return f + def claim(self,f,asset,attempt): + return self.command('claim',{**f,'asset_id':asset,'asset_attempt':attempt,'request_id':str(uuid.uuid4())}) + def running(self): + s,_=self.approve_v2();f=self.bind(s);asset=s['assets'][0]['id'];c=self.claim(f,asset,0) + return {**f,'asset_id':asset,'asset_attempt':c['asset']['attempt'],'claim_id':c['asset']['claim_id']} + def snapshot(self): + return self.sql(f"SELECT jsonb_build_object('job',(SELECT to_jsonb(j) FROM release_pack_jobs j WHERE id='{self.job}'),'assets',(SELECT jsonb_agg(to_jsonb(a) ORDER BY id) FROM release_pack_assets a WHERE job_id='{self.job}'),'wallet',(SELECT jsonb_agg(to_jsonb(w) ORDER BY id) FROM credits_usage w WHERE account_id='{self.owner}'),'usage',(SELECT coalesce(jsonb_agg(to_jsonb(u) ORDER BY id),'[]') FROM usage_events u WHERE account_id='{self.owner}'))").stdout.strip() + def retry(self,old): + self.command('finish_asset',{**old,'status':'failed','error':'confirmed failure'}) + self.command('settle',old) + q={'id':str(uuid.uuid4()),'revision':1,'credits':100,'maximum_usd':.01,'expires_at':'2099-01-01T00:00:00Z','retry_asset_id':old['asset_id'],'execution_config':{'models':{'image':'fixture','image_edit':'fixture-edit','motion':'fixture-motion'},'rates':{'image_usd_per_unit':.004,'credits_per_usd':10000}}} + self.command('estimate',{'expected_revision':1,'asset_attempt':1,'retry_asset_id':old['asset_id'],'quote':q}) + s=self.command('approve',{'request_id':str(uuid.uuid4()),'quote_id':q['id'],'expected_revision':1,'approved_max_usd':.01,'source_execution_id':old['execution_id'],'source_execution_version':old['execution_version']}) + f=self.bind(s,'run-2'); c=self.claim(f,old['asset_id'],1) + return {**f,'asset_id':old['asset_id'],'asset_attempt':2,'claim_id':c['asset']['claim_id']} + def test_stale_all_writers_and_before_expiry(self): + old=self.running();fresh=self.retry(old) + for expired in [False, True]: + if expired: + self.sql(f"UPDATE release_pack_jobs SET hold_expires_at=now()-interval '1 second' WHERE id='{self.job}'") + for action,payload in [('phase',{'phase':'image_submission_0','value':{'state':'accepted','request_id':'old','charge_credits':40}}),('finish_asset',{'status':'completed','files':[{'storage_key':'old'}],'caption':'old','qa':{'passed':True}}),('context',{'context_version':0,'value':{'old':True}}),('run',{}),('settle',{}),('claim',{'request_id':str(uuid.uuid4())}),('dispatch_uncertain',{'error':'old'})]: + before=self.snapshot();r=self.command(action,{**old,**payload},succeeds=False) + self.assertNotEqual(r.returncode,0,action);self.assertEqual(before,self.snapshot(),action) + def test_transition_replay_payload_conflict_and_legacy_isolation(self): + s,p=self.approve_v2();before=self.snapshot();replayed=self.command('approve',p) + self.assertFalse(replayed['dispatch']);self.assertEqual(before,self.snapshot()) + self.assertNotEqual(self.command('approve',{**p,'approved_max_usd':1},succeeds=False).returncode,0) + self.assertNotEqual(self.command('read',legacy=True,succeeds=False).returncode,0) + self.assertEqual(s['job']['protocol'],2) + def test_claim_run_context_finish_settle_replay(self): + s,_=self.approve_v2();f=self.bind(s);asset=s['assets'][0]['id'] + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + rows=list(pool.map(lambda _:self.claim(f,asset,0),range(4))) + self.assertEqual(sum(r['claimed'] for r in rows),1) + c=next(r for r in rows if r['claimed']);af={**f,'asset_id':asset,'asset_attempt':1,'claim_id':c['asset']['claim_id']} + self.assertNotEqual(self.command('run',{**f,'workflow_run_id':'other'},succeeds=False).returncode,0) + self.command('context',{**f,'context_version':0,'value':{'state':'complete'}}) + self.assertNotEqual(self.command('context',{**f,'context_version':0,'value':{'stale':True}},succeeds=False).returncode,0) + done={**af,'status':'completed','files':[{'storage_key':'fixture'}],'caption':'fixture','qa':{'passed':True}} + self.command('finish_asset',done);before=self.snapshot();self.command('finish_asset',done);self.assertEqual(before,self.snapshot()) + self.assertNotEqual(self.command('finish_asset',{**done,'caption':'changed'},succeeds=False).returncode,0) + self.command('settle',f);before=self.snapshot();self.command('settle',f);self.assertEqual(before,self.snapshot()) + def test_real_registration_receipt_and_resume_fence(self): + f=self.running();v={'state':'submitting','model':'fixture','input_sha256':'a'*64,'charge_credits':40} + s=self.command('phase',{**f,'phase':'image_submission_0','value':v});op=s['operation_id'] + self.observe(op,f,{'kind':'accepted','request_id':'original'}) + self.command('phase',{**f,'phase':'image_submission_0','value':{**v,'state':'accepted','request_id':'original'}}) + self.command('finish_asset',{**f,'status':'uncertain','error':'lost result'});self.command('settle',f) + j=self.command('read')['job'];p={'request_id':str(uuid.uuid4()),'quote_id':j['quote']['id'],'expected_revision':1,'approved_max_usd':.006,'asset_id':f['asset_id'],'asset_attempt':1,'source_claim_id':f['claim_id'],'source_execution_id':f['execution_id'],'source_execution_version':f['execution_version']} + s=self.command('resume',p);newf=self.bind(s,'resumed');c=self.claim(newf,f['asset_id'],1);newf={**newf,'asset_id':f['asset_id'],'asset_attempt':1,'claim_id':c['asset']['claim_id']} + self.assertEqual(c['asset']['attempt'],1) + before=self.snapshot();self.assertNotEqual(self.command('finish_asset',{**f,'status':'failed'},succeeds=False).returncode,0);self.assertEqual(before,self.snapshot()) + self.observe(op,newf,{'kind':'completed','request_id':'original','output_sha256':'b'*64}) + self.assertEqual(before,self.snapshot()) + self.command('finish_asset',{**newf,'status':'uncertain','error':'again'});self.command('settle',newf) + before=self.snapshot();self.assertNotEqual(self.command('resume',{**p,'request_id':str(uuid.uuid4())},succeeds=False).returncode,0);self.assertEqual(before,self.snapshot()) + self.assertFalse(self.command('resume',p)['dispatch']) + def observe(self,op,f,payload,succeeds=True,actor=None): + data=json.dumps(payload).replace("'","''");identity=json.dumps({'execution_id':f['execution_id'],'claim_id':f['claim_id']}) + r=self.sql(f"SET ROLE service_role; SELECT append_pack_provider_observation('{actor or self.owner}',null,'{op}','{identity}'::jsonb,'{data}'::jsonb)",succeeds=succeeds) + return json.loads(r.stdout) if succeeds else r + +if __name__=='__main__': + names=[x for x in FencedPacks.__dict__ if x.startswith('test_')] + result=unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite(FencedPacks(x) for x in names)) + raise SystemExit(not result.wasSuccessful()) diff --git a/proposals/pack-worker-fence-2269/test_extended.py b/proposals/pack-worker-fence-2269/test_extended.py new file mode 100644 index 0000000..bed1406 --- /dev/null +++ b/proposals/pack-worker-fence-2269/test_extended.py @@ -0,0 +1,108 @@ +"""Additional actual proposal checks; no hosted state.""" +import concurrent.futures +import json +import unittest +import uuid +from test_contract import FencedPacks, ReleasePackJobs + +class Extended(FencedPacks): + def register(self,f,phase='image_submission_0'): + v={'state':'submitting','model':'fixture','input_sha256':'a'*64,'charge_credits':40} + return self.command('phase',{**f,'phase':phase,'value':v})['operation_id'],v + def test_actual_late_receipt_survives_stale_refusal_and_conflict_races(self): + f=self.running();op,v=self.register(f) + self.observe(op,f,{'kind':'accepted','request_id':'original'}) + self.command('phase',{**f,'phase':'image_submission_0','value':{**v,'state':'accepted','request_id':'original'}}) + self.command('finish_asset',{**f,'status':'uncertain'});self.command('settle',f) + j=self.command('read')['job'] + resumed=self.command('resume',{'request_id':str(uuid.uuid4()),'quote_id':j['quote']['id'],'expected_revision':1,'approved_max_usd':.006,'asset_id':f['asset_id'],'asset_attempt':1,'source_claim_id':f['claim_id'],'source_execution_id':f['execution_id'],'source_execution_version':f['execution_version']}) + newf=self.bind(resumed,'run-resume');self.claim(newf,f['asset_id'],1) + before=self.snapshot() + self.observe(op,f,{'kind':'completed','request_id':'original','output_sha256':'b'*64}) + self.assertNotEqual(self.command('phase',{**f,'phase':'image_submission_0','value':{**v,'state':'complete','request_id':'original','data':{'old':'result'}}},succeeds=False).returncode,0) + self.assertEqual(before,self.snapshot()) + with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool: + rows=list(pool.map(lambda _:self.observe(op,f,{'kind':'accepted','request_id':'conflicting'}),range(6))) + self.assertTrue(all(x['state']=='conflict' for x in rows)) + self.assertEqual(self.sql(f"SELECT count(*) FROM release_pack_provider_observations WHERE operation_id='{op}'").stdout.strip(),'3') + self.assertEqual(self.observe(op,f,{'kind':'accepted','request_id':'third'})['state'],'conflict_unretained_capacity') + self.assertEqual(before,self.snapshot()) + def test_owner_pages_redaction_and_privilege_denials(self): + self.asset_kind='carousel';f=self.running();op,_=self.register(f);self.register(f,'image_submission_1') + self.observe(op,f,{'kind':'accepted','request_id':'original'}) + for bad in [{'kind':'accepted','request_id':'original','raw':'secret'}, {'kind':'accepted','request_id':'x'*1100}]: + self.assertNotEqual(self.observe(op,f,bad,succeeds=False).returncode,0) + def page(actor,after='null'): + return self.sql(f"SET ROLE service_role; SELECT read_pack_provider_observations('{actor}',null,'{self.job}',{after},1)",succeeds=False) + first=json.loads(page(self.owner).stdout);self.assertEqual(len(first['operations']),1) + second=json.loads(page(self.owner,"'"+first['next_cursor']+"'").stdout);self.assertIsNone(second['next_cursor']) + self.assertNotEqual(page(str(uuid.uuid4())).returncode,0) + self.assertNotEqual(page(self.owner,"'"+str(uuid.uuid4())+"'").returncode,0) + for role in ['anon','authenticated','service_role']: + self.assertNotEqual(self.sql(f'SET ROLE {role}; DELETE FROM release_pack_provider_observations',succeeds=False).returncode,0) + self.assertEqual(self.sql("SELECT has_table_privilege('pack_receipt_port','release_pack_jobs','UPDATE'),has_table_privilege('pack_receipt_port','credits_usage','UPDATE'),has_table_privilege('pack_receipt_port','usage_events','INSERT')").stdout.strip(),'f|f|f') + def test_unobserved_charge_and_direct_complete_refused_atomically(self): + f=self.running();op,v=self.register(f);before=self.snapshot() + self.assertNotEqual(self.command('phase',{**f,'phase':'image_submission_0','value':{**v,'state':'accepted','request_id':'unretained'}},succeeds=False).returncode,0) + self.assertEqual(before,self.snapshot()) + self.assertNotEqual(self.command('finish_asset',{**f,'status':'failed'},succeeds=False).returncode,0) + self.observe(op,f,{'kind':'accepted','request_id':'accepted'}) + self.observe(op,f,{'kind':'completed','request_id':'accepted'}) + self.assertNotEqual(self.command('phase',{**f,'phase':'image_submission_0','value':{**v,'state':'complete','request_id':'accepted','data':{}}},succeeds=False).returncode,0) + self.assertEqual(before,self.snapshot()) + payload={**f,'phase':'image_submission_0','value':{**v,'state':'accepted','request_id':'accepted'}} + self.command('phase',payload);before=self.snapshot();self.command('phase',payload);self.assertEqual(before,self.snapshot()) + self.assertEqual(self.sql(f"SELECT count(*) FROM usage_events WHERE account_id='{self.owner}'").stdout.strip(),'1') + def test_legacy_job_refuses_v2_without_adoption(self): + self.command('prepare',{'idempotency_key':self.job,'fingerprint':'a'*64,'brief':{},'deliverables':{},'status':'ready_to_quote'},legacy=True) + before=self.snapshot();self.assertNotEqual(self.command('read',succeeds=False).returncode,0);self.assertEqual(before,self.snapshot()) + + def test_claim_replay_and_concurrent_intent_grant_only_once(self): + saved,_=self.approve_v2();f=self.bind(saved);asset=saved['assets'][0]['id'] + payload={**f,'asset_id':asset,'asset_attempt':0,'request_id':str(uuid.uuid4())} + claimed=self.command('claim',payload);before=self.snapshot() + replay=self.command('claim',payload) + self.assertFalse(replay['claimed']);self.assertTrue(replay['replayed']);self.assertEqual(before,self.snapshot()) + f={**f,'asset_id':asset,'asset_attempt':1,'claim_id':claimed['asset']['claim_id']} + payload={**f,'phase':'image_submission_0','value':{'state':'submitting','model':'fixture','input_sha256':'a'*64,'charge_credits':40}} + with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool: + rows=list(pool.map(lambda _:self.command('phase',payload),range(6))) + self.assertEqual(sum(r['operation_created'] for r in rows),1) + self.assertEqual(len({r['operation_id'] for r in rows}),1) + before=self.snapshot() + self.assertNotEqual(self.command('phase',{**payload,'value':{**payload['value'],'input_sha256':'b'*64}},succeeds=False).returncode,0) + self.assertEqual(before,self.snapshot()) + def test_current_expiry_refunds_once_and_missing_fences_never_trigger_it(self): + f=self.running() + self.sql(f"UPDATE release_pack_jobs SET hold_expires_at=now()-interval '1 second' WHERE id='{self.job}'") + before=self.snapshot() + for missing in ['execution_id','execution_version','expected_revision','workflow_run_id','claim_id','asset_attempt']: + payload={**f,'status':'completed'};del payload[missing] + self.assertNotEqual(self.command('finish_asset',payload,succeeds=False).returncode,0) + self.assertEqual(before,self.snapshot()) + expired=self.command('settle',f);self.assertTrue(expired['command_expired']) + settled=self.snapshot();self.command('read');self.command('read');self.assertEqual(settled,self.snapshot()) + def test_receipt_foreign_identity_and_invalid_page_leave_state_unchanged(self): + f=self.running();op,_=self.register(f);before=self.snapshot() + for identity in [{**f,'claim_id':str(uuid.uuid4())},{**f,'execution_id':str(uuid.uuid4())}]: + self.assertNotEqual(self.observe(op,identity,{'kind':'accepted','request_id':'original'},succeeds=False).returncode,0) + self.assertNotEqual(self.observe(op,f,{'kind':'accepted','request_id':'original'},actor=str(uuid.uuid4()),succeeds=False).returncode,0) + for limit in ['0','101','null']: + self.assertNotEqual(self.sql(f"SET ROLE service_role; SELECT read_pack_provider_observations('{self.owner}',null,'{self.job}',null,{limit})",succeeds=False).returncode,0) + self.assertEqual(self.sql(f"SELECT count(*) FROM release_pack_provider_observations WHERE operation_id='{op}'").stdout.strip(),'0') + self.assertEqual(before,self.snapshot()) + +class LegacyCompatibility(ReleasePackJobs): + @classmethod + def setUpClass(cls): + super().setUpClass() + for name in ['schema.sql','command.sql','receipts.sql']: + cls.sql((__import__('pathlib').Path(__file__).parent/name).read_text()) + +if __name__=='__main__': + suite=unittest.TestSuite() + for cls in [FencedPacks,Extended]: + suite.addTests(cls(x) for x in cls.__dict__ if x.startswith('test_')) + suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(LegacyCompatibility)) + result=unittest.TextTestRunner(verbosity=2).run(suite) + raise SystemExit(not result.wasSuccessful()) From 186eea9d5b03ccb03e3b10a82e10ca2875d35c5d Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:55:46 -0500 Subject: [PATCH 2/3] Preserve Pack protocol ownership across concurrent creation --- proposals/pack-worker-fence-2269/CONTRACT.md | 4 +- proposals/pack-worker-fence-2269/README.md | 6 +- proposals/pack-worker-fence-2269/command.sql | 17 ++-- proposals/pack-worker-fence-2269/schema.sql | 8 +- .../test_creation_race.py | 94 +++++++++++++++++++ .../pack-worker-fence-2269/test_extended.py | 3 +- 6 files changed, 118 insertions(+), 14 deletions(-) create mode 100644 proposals/pack-worker-fence-2269/test_creation_race.py diff --git a/proposals/pack-worker-fence-2269/CONTRACT.md b/proposals/pack-worker-fence-2269/CONTRACT.md index 1a48ac9..cda5056 100644 --- a/proposals/pack-worker-fence-2269/CONTRACT.md +++ b/proposals/pack-worker-fence-2269/CONTRACT.md @@ -4,7 +4,7 @@ Implemented as inactive proposal SQL; see [README](README.md) for qualification ## Identity and ownership -Existing jobs default to protocol 1. A new protocol-2 prepare is explicit; an existing legacy job cannot be adopted. A guarded legacy wrapper rejects protocol-2 jobs, including idempotency-key resolution, and retains the original command for protocol 1. The original command is renamed to a private internal function; service-role direct Pack INSERT/UPDATE/DELETE and internal-function execution are revoked. Current API Pack persistence uses the RPC, not raw table DML. A future rollout must inventory other callers before applying those grants. +Existing jobs default to protocol 1. A new protocol-2 prepare is explicit; an existing legacy job cannot be adopted. Both public new-prepare paths take the same owner/idempotency-key transaction advisory lock before lookup/insertion. Protocol 2 is assigned directly on INSERT; conflict resolution locks the actual owner/key winner and verifies protocol and fingerprint. No UPDATE promotes a protocol-1 row. These two protections prevent an uncommitted legacy insert from becoming a protocol-2 job after a unique-index wait. A guarded legacy wrapper rejects protocol-2 jobs, including idempotency-key resolution, and retains the original command for protocol 1. The original command is renamed to a private internal function; service-role direct Pack INSERT/UPDATE/DELETE and internal-function execution are revoked. Current API Pack persistence uses the RPC, not raw table DML. A future rollout must inventory other callers before applying those grants. Each newly accepted approval/resume creates an execution UUID and increasing job execution version. The transition request UUID and complete JSONB payload are retained. Source execution/version and creative revision are compare-and-set inputs. Resume also compares the original asset attempt/claim. Exact accepted transition replay returns its recorded result with `dispatch=false`; conflicting payload reuse refuses. Replay lookup precedes source-generation comparison, so an old accepted transition can be inspected without regaining write authority. An unseen stale resume cannot rotate a newer generation. @@ -42,6 +42,6 @@ The Workflow SDK reconstructs step errors, so the protocol-2 workflow stops on * ## Evidence and limits -Actual PostgreSQL checks cover stale writers before/after expiry, same-attempt resume, transition/claim/completion/settlement replay and conflict, concurrent claim/intent registration, context CAS, atomic charge refusal, receipt retention/access/bounds and the existing legacy controls. The paired API-to-PG fixture covers real claim/intent/journal/charge/completion, claim replay refusal, mismatched actual Workflow identity, foundation-dependent image-edit and a committed acceptance surviving an actual phase refusal without catch finish or charge. Compiled local SDK checks cover step failure, lost settlement acknowledgement and terminal replay, with networking denied. +Actual PostgreSQL checks include deterministic two-connection legacy-first/protocol-2-first/same-protocol creation races, then cover stale writers before/after expiry, same-attempt resume, transition/claim/completion/settlement replay and conflict, concurrent claim/intent registration, context CAS, atomic charge refusal, receipt retention/access/bounds and the existing legacy controls. The paired API-to-PG fixture covers real claim/intent/journal/charge/completion, claim replay refusal, mismatched actual Workflow identity, foundation-dependent image-edit and a committed acceptance surviving an actual phase refusal without catch finish or charge. Compiled local SDK checks cover step failure, lost settlement acknowledgement and terminal replay, with networking denied. This does not cancel already-issued provider work, guarantee an external I/O fence stays current, attest provider responses, recover a response lost before durable recording, establish cache/QA freshness, or make legacy executions safe. Unknown intent cannot be automatically resubmitted. Operator reconciliation, rollout, hosted SQL, provider execution, paid runtime, renderer activation and customer outcomes remain outside this contribution. diff --git a/proposals/pack-worker-fence-2269/README.md b/proposals/pack-worker-fence-2269/README.md index 0f58037..bd39a54 100644 --- a/proposals/pack-worker-fence-2269/README.md +++ b/proposals/pack-worker-fence-2269/README.md @@ -9,7 +9,7 @@ The paired API branch is `codex/pack-worker-fence-2269` in `recoupable/api`. Its - `schema.sql`: protocol ownership, execution/claim history, the two receipt tables and guarded legacy wrapper. - `command.sql`: generation, workflow, claim and context fences; intent registration and existing-accounting composition. - `receipts.sql`: separately committed, redacted observation append/read ports. -- `test_contract.py` and `test_extended.py`: actual command calls on disposable PostgreSQL, using the repository's existing synthetic fixture. +- `test_contract.py`, `test_creation_race.py` and `test_extended.py`: actual command calls on disposable PostgreSQL, using the repository's existing synthetic fixture. - `CONTRACT.md` and `RECEIPTS.md`: implemented behavior and remaining boundaries. Run with PostgreSQL 17 tools on PATH: @@ -18,10 +18,12 @@ Run with PostgreSQL 17 tools on PATH: PYTHONDONTWRITEBYTECODE=1 python3 proposals/pack-worker-fence-2269/test_extended.py ``` -The fixture applies the existing Pack schema and then these three proposal files, in the order above, only to its owned temporary Unix-socket cluster. It uses no TCP listener or hosted credentials and cleans up its own cluster. The suite covers 11 protocol-2 cases and the existing 11 legacy controls. The paired API provides an additional runner that exercises real API production helpers against these exact SQL files; provider, media and caption effects are synthetic. Its compiled Workflow SDK fixture verifies failure/replay behavior separately. +The fixture applies the existing Pack schema and then these three proposal files, in the order above, only to its owned temporary Unix-socket cluster. It uses no TCP listener or hosted credentials and cleans up its own cluster. The suite covers 14 protocol-2/creation cases and the existing 11 legacy controls. The paired API provides an additional runner that exercises real API production helpers against these exact SQL files; provider, media and caption effects are synthetic. Its compiled Workflow SDK fixture verifies failure/replay behavior separately. ## Delivery and activation boundary Baseline: Database `1b892dd2671ba7e15e0c44b690fbfbe515b59fbd`; paired API `bb542a9924ba531e422b3cd229e75a8bdb1892da`. No historical migration is changed. Migration filename/ownership work in Database PRs #125/#135 and reservation composition in #134/#135 remain separate. Existing Pack accounting remains the implementation called internally; no new wallet, rates or reservation model is introduced. Before any rollout, a separately reviewed change must reconcile the current database head and migration ownership, convert this proposal into an ordered migration, verify deployed schema/permissions, inventory and drain or reconcile legacy in-flight executions, qualify the intended authenticated runtime, and explicitly admit new protocol-2 launches. This PR does none of those things. The renderer and foundation/scheduler candidates remain unwired. App 2269 remains open. + +Creation-race repair: both public prepare paths take the same owner/idempotency-key transaction lock. Protocol 2 is assigned on INSERT, never by promoting the conflict winner. The locked winner must already be protocol 2 with matching fingerprint. Three deterministic two-connection regressions cover legacy-first, protocol-2-first and same-protocol contention; they wait for an observed PostgreSQL lock before committing the winner. The legacy-first case failed before the repair by converting the committed legacy row; all three pass after it. Legacy read/replay/estimate remain usable. diff --git a/proposals/pack-worker-fence-2269/command.sql b/proposals/pack-worker-fence-2269/command.sql index 46345de..d76cd0e 100644 --- a/proposals/pack-worker-fence-2269/command.sql +++ b/proposals/pack-worker-fence-2269/command.sql @@ -11,14 +11,17 @@ BEGIN owner:=public.authorize_pack_actor(p_actor,p_org); IF jsonb_typeof(p_input) IS DISTINCT FROM 'object' THEN RAISE EXCEPTION 'Invalid Pack input'; END IF; IF p_action='prepare' AND NOT(p_input ? 'expected_revision') THEN - -- Lock new identity/idempotency resolution before changing protocol ownership. + -- Shared with legacy prepare. Protocol ownership is set on INSERT, never promoted. PERFORM pg_advisory_xact_lock(hashtextextended(owner::text||':'||(p_input->>'idempotency_key'),0)); - IF EXISTS(SELECT 1 FROM public.release_pack_jobs WHERE owner_id=owner AND idempotency_key=p_input->>'idempotency_key' AND protocol<>2) THEN - RAISE EXCEPTION 'Legacy Pack cannot be adopted'; END IF; - result:=public.release_pack_command_legacy_internal(p_actor,p_org,p_action,p_job,p_input); - p_job:=(result->'job'->>'id')::uuid; - UPDATE public.release_pack_jobs SET protocol=2 WHERE id=p_job; - RETURN public.release_pack_command_legacy_internal(p_actor,p_org,'read',p_job,'{}'); + INSERT INTO public.release_pack_jobs(id,owner_id,created_by,idempotency_key,fingerprint,brief,deliverables,status,sufficient_brief_at,protocol) + VALUES(p_job,owner,p_actor,p_input->>'idempotency_key',p_input->>'fingerprint',p_input->'brief',p_input->'deliverables',p_input->>'status',CASE WHEN p_input->>'status'='ready_to_quote' THEN now() END,2) + ON CONFLICT(owner_id,idempotency_key) DO NOTHING; + -- Resolve and lock the actual winner, including after a unique-index wait. + SELECT * INTO STRICT j FROM public.release_pack_jobs + WHERE owner_id=owner AND idempotency_key=p_input->>'idempotency_key' FOR UPDATE; + IF j.protocol<>2 THEN RAISE EXCEPTION 'Legacy Pack cannot be adopted'; END IF; + IF j.fingerprint IS DISTINCT FROM p_input->>'fingerprint' THEN RAISE EXCEPTION 'Idempotency key conflicts with different inputs'; END IF; + RETURN public.release_pack_command_legacy_internal(p_actor,p_org,'read',j.id,'{}'); END IF; SELECT * INTO STRICT j FROM public.release_pack_jobs WHERE id=p_job AND owner_id=owner FOR UPDATE; IF j.protocol<>2 THEN RAISE EXCEPTION 'Legacy Pack cannot be adopted'; END IF; diff --git a/proposals/pack-worker-fence-2269/schema.sql b/proposals/pack-worker-fence-2269/schema.sql index 169d605..24bfb61 100644 --- a/proposals/pack-worker-fence-2269/schema.sql +++ b/proposals/pack-worker-fence-2269/schema.sql @@ -61,9 +61,13 @@ END $$; REVOKE ALL ON FUNCTION public.authorize_pack_actor(uuid,uuid) FROM PUBLIC,anon,authenticated,service_role; CREATE FUNCTION public.release_pack_command(p_actor uuid,p_org uuid,p_action text,p_job uuid,p_input jsonb DEFAULT '{}') RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$ -DECLARE result jsonb; +DECLARE result jsonb; owner uuid; BEGIN - PERFORM public.authorize_pack_actor(p_actor,p_org); + owner:=public.authorize_pack_actor(p_actor,p_org); + -- Both protocols serialize the same creation identity before looking up or inserting. + IF p_action='prepare' AND NOT(p_input ? 'expected_revision') THEN + PERFORM pg_advisory_xact_lock(hashtextextended(owner::text||':'||(p_input->>'idempotency_key'),0)); + END IF; IF EXISTS(SELECT 1 FROM public.release_pack_jobs WHERE id=p_job AND protocol=2) THEN RAISE EXCEPTION 'Protocol 2 requires a fenced caller'; END IF; result:=public.release_pack_command_legacy_internal(p_actor,p_org,p_action,p_job,p_input); diff --git a/proposals/pack-worker-fence-2269/test_creation_race.py b/proposals/pack-worker-fence-2269/test_creation_race.py new file mode 100644 index 0000000..aa7fe3d --- /dev/null +++ b/proposals/pack-worker-fence-2269/test_creation_race.py @@ -0,0 +1,94 @@ +"""Two real connections race protocol ownership; no mocked command or seeded job.""" +import concurrent.futures +import json +import select +import subprocess +import time +import unittest +import uuid +from test_contract import FencedPacks + + +class CreationRace(FencedPacks): + def prepare_sql(self, protocol, job, key, fingerprint=None): + name = 'release_pack_command_v2' if protocol == 2 else 'release_pack_command' + payload = json.dumps({'idempotency_key': key, 'fingerprint': fingerprint or 'a' * 64, + 'brief': {}, 'deliverables': {}, 'status': 'ready_to_quote'}) + return f"SELECT {name}('{self.owner}',null,'prepare','{job}','{payload}'::jsonb);" + + def race(self, winner_protocol, loser_protocol): + key = str(uuid.uuid4()) + winner_job, loser_job = str(uuid.uuid4()), str(uuid.uuid4()) + winner = subprocess.Popen(['psql', '-h', self.tmp.name, '-p', '55476', '-d', 'postgres', + '-XAtq', '-v', 'ON_ERROR_STOP=1'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + winner.stdin.write("BEGIN; SET LOCAL statement_timeout='10s'; SET ROLE service_role; " + self.prepare_sql(winner_protocol, winner_job, key) + '\n') + winner.stdin.flush() + self.assertTrue(select.select([winner.stdout], [], [], 10)[0], 'winner did not prepare') + first = json.loads(winner.stdout.readline()) + self.assertEqual(first['job']['protocol'], winner_protocol) + tag = 'pack_creation_' + uuid.uuid4().hex + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + pending = pool.submit(self.sql, f"SET statement_timeout='10s'; SET application_name='{tag}'; SET ROLE service_role; " + self.prepare_sql(loser_protocol, loser_job, key), False) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + waiting = self.sql(f"SELECT wait_event_type||':'||wait_event FROM pg_stat_activity WHERE application_name='{tag}' AND wait_event_type='Lock'").stdout.strip() + if waiting: + break + self.assertFalse(pending.done(), 'contender escaped the uncommitted winner') + time.sleep(.01) + else: + self.fail('contender did not reach an actual PostgreSQL lock') + winner.stdin.write('COMMIT;\n') + winner.stdin.flush() + result = pending.result(timeout=10) + print(json.dumps({'winner_protocol': winner_protocol, 'contender_protocol': loser_protocol, + 'observed_wait': waiting, 'contender_exit': result.returncode, + 'retained_protocol': self.sql(f"SELECT protocol FROM release_pack_jobs WHERE id='{winner_job}'").stdout.strip()}), flush=True) + return first, result, waiting, key, loser_job + finally: + if winner.poll() is None: + winner.communicate('ROLLBACK;\n\\q\n', timeout=10) + else: + winner.communicate(timeout=10) + + def test_legacy_winner_stays_legacy_and_protocol2_loser_refuses(self): + first, result, waiting, key, _ = self.race(1, 2) + self.assertNotEqual(result.returncode, 0, f'protocol2 adopted legacy after {waiting}: {result.stdout}') + self.assertIn('Legacy Pack cannot be adopted', result.stderr) + self.job = first['job']['id'] + self.assertEqual(self.command('read', legacy=True)['job']['protocol'], 1) + before = self.snapshot() + replay = self.sql('SET ROLE service_role; ' + self.prepare_sql(1, str(uuid.uuid4()), key)) + self.assertEqual(json.loads(replay.stdout)['job']['id'], self.job) + self.assertEqual(before, self.snapshot()) + self.assertNotEqual(self.command('read', succeeds=False).returncode, 0) + quote = {'id': str(uuid.uuid4()), 'revision': 1, 'credits': 100, 'maximum_usd': .01, 'expires_at': '2099-01-01T00:00:00Z'} + quoted = self.command('estimate', {'quote': quote, 'assets': [{'kind': 'artwork', 'ordinal': 0, 'credits': 100, 'maximum_usd': .01}]}, legacy=True) + self.assertEqual(quoted['job']['protocol'], 1) + self.assertEqual(quoted['job']['status'], 'awaiting_approval') + + def test_protocol2_winner_rejects_legacy_and_keeps_owner_guard_and_replay(self): + first, result, _, key, _ = self.race(2, 1) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Protocol 2 requires a fenced caller', result.stderr) + self.job = first['job']['id'] + before = self.snapshot() + replay = self.sql('SET ROLE service_role; ' + self.prepare_sql(2, str(uuid.uuid4()), key)) + self.assertEqual(json.loads(replay.stdout)['job']['id'], self.job) + self.assertEqual(self.command('read')['job']['protocol'], 2) + self.assertNotEqual(self.command('read', actor=str(uuid.uuid4()), succeeds=False).returncode, 0) + self.assertNotEqual(self.sql('SET ROLE service_role; ' + self.prepare_sql(2, str(uuid.uuid4()), key, 'b' * 64), succeeds=False).returncode, 0) + self.assertEqual(before, self.snapshot()) + + def test_two_protocol2_creators_replay_one_owned_job(self): + first, result, _, _, _ = self.race(2, 2) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)['job']['id'], first['job']['id']) + self.assertEqual(self.sql(f"SELECT count(*) FROM release_pack_jobs WHERE owner_id='{self.owner}'").stdout.strip(), '1') + + +if __name__ == '__main__': + suite = unittest.TestSuite(CreationRace(name) for name in CreationRace.__dict__ if name.startswith('test_')) + raise SystemExit(not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful()) diff --git a/proposals/pack-worker-fence-2269/test_extended.py b/proposals/pack-worker-fence-2269/test_extended.py index bed1406..8f20d6a 100644 --- a/proposals/pack-worker-fence-2269/test_extended.py +++ b/proposals/pack-worker-fence-2269/test_extended.py @@ -4,6 +4,7 @@ import unittest import uuid from test_contract import FencedPacks, ReleasePackJobs +from test_creation_race import CreationRace class Extended(FencedPacks): def register(self,f,phase='image_submission_0'): @@ -101,7 +102,7 @@ def setUpClass(cls): if __name__=='__main__': suite=unittest.TestSuite() - for cls in [FencedPacks,Extended]: + for cls in [FencedPacks,Extended,CreationRace]: suite.addTests(cls(x) for x in cls.__dict__ if x.startswith('test_')) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(LegacyCompatibility)) result=unittest.TextTestRunner(verbosity=2).run(suite) From c9d87b85ca53339f4084f8e62ed7c7ac5115f491 Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:40:52 -0500 Subject: [PATCH 3/3] fix(packs): validate worker actions before expiry and completion --- .github/workflows/release-pack-jobs.yml | 4 + proposals/pack-worker-fence-2269/CONTRACT.md | 6 ++ proposals/pack-worker-fence-2269/RECEIPTS.md | 4 +- proposals/pack-worker-fence-2269/command.sql | 38 +++++--- proposals/pack-worker-fence-2269/receipts.sql | 1 + proposals/pack-worker-fence-2269/schema.sql | 1 + .../test_expiry_fence.py | 79 +++++++++++++++ .../pack-worker-fence-2269/test_extended.py | 4 +- .../test_review_regressions.py | 96 +++++++++++++++++++ 9 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 proposals/pack-worker-fence-2269/test_expiry_fence.py create mode 100644 proposals/pack-worker-fence-2269/test_review_regressions.py diff --git a/.github/workflows/release-pack-jobs.yml b/.github/workflows/release-pack-jobs.yml index b2155ee..b5f89d1 100644 --- a/.github/workflows/release-pack-jobs.yml +++ b/.github/workflows/release-pack-jobs.yml @@ -18,3 +18,7 @@ jobs: run: | export PATH="/usr/lib/postgresql/16/bin:$PATH" python3 -m unittest discover -s tests -p 'test_release_pack_jobs.py' + - name: Verify inactive fenced proposal and creation races + run: | + export PATH="/usr/lib/postgresql/16/bin:$PATH" + PYTHONDONTWRITEBYTECODE=1 python3 proposals/pack-worker-fence-2269/test_extended.py diff --git a/proposals/pack-worker-fence-2269/CONTRACT.md b/proposals/pack-worker-fence-2269/CONTRACT.md index cda5056..898f3d5 100644 --- a/proposals/pack-worker-fence-2269/CONTRACT.md +++ b/proposals/pack-worker-fence-2269/CONTRACT.md @@ -45,3 +45,9 @@ The Workflow SDK reconstructs step errors, so the protocol-2 workflow stops on * Actual PostgreSQL checks include deterministic two-connection legacy-first/protocol-2-first/same-protocol creation races, then cover stale writers before/after expiry, same-attempt resume, transition/claim/completion/settlement replay and conflict, concurrent claim/intent registration, context CAS, atomic charge refusal, receipt retention/access/bounds and the existing legacy controls. The paired API-to-PG fixture covers real claim/intent/journal/charge/completion, claim replay refusal, mismatched actual Workflow identity, foundation-dependent image-edit and a committed acceptance surviving an actual phase refusal without catch finish or charge. Compiled local SDK checks cover step failure, lost settlement acknowledgement and terminal replay, with networking denied. This does not cancel already-issued provider work, guarantee an external I/O fence stays current, attest provider responses, recover a response lost before durable recording, establish cache/QA freshness, or make legacy executions safe. Unknown intent cannot be automatically resubmitted. Operator reconciliation, rollout, hosted SQL, provider execution, paid runtime, renderer activation and customer outcomes remain outside this contribution. + +Post-review fence ordering: unknown/null actions, context CAS, pending-attempt CAS and claim replay/conflict are resolved before expiry recovery. A bound `dispatch_uncertain` call must carry its bound workflow ID; only an unbound dispatch can report uncertainty without one. Expired explicit settlement stores the original caller payload and the flagged result for exact replay. + +Submission intent cannot include `request_id`; only a durable accepted observation permits a charged accepted phase. Neither failed nor completed asset status can close an incomplete submission journal; uncertainty remains available for reconciliation. + +The checked-in Release pack transactions CI runs `test_extended.py`, including protocol creation races and post-review regressions, against disposable PostgreSQL only. diff --git a/proposals/pack-worker-fence-2269/RECEIPTS.md b/proposals/pack-worker-fence-2269/RECEIPTS.md index 1587910..91e67c6 100644 --- a/proposals/pack-worker-fence-2269/RECEIPTS.md +++ b/proposals/pack-worker-fence-2269/RECEIPTS.md @@ -12,7 +12,7 @@ The command bounds phase slots: image 0–3 for carousel, otherwise image 0, and ## Bounded observations -`release_pack_provider_observations` has primary key `(operation_id, kind)`, where kind is accepted, completed, failed or first conflict; at most four rows per operation. Payloads are at most 1024 bytes. Accepted/completed/failed accept only: +`release_pack_provider_observations` has primary key `(operation_id, kind)`, where kind is accepted, completed, failed or `conflict` (first conflict); at most four rows per operation. Payloads are at most 1024 bytes. Accepted/completed/failed accept only: - `kind` and `request_id` (1–200 ASCII letters, digits, underscore or hyphen). - Optional `output_sha256`, 64 lowercase hexadecimal characters, completed only. @@ -44,3 +44,5 @@ The owner-scoped read port validates job and cursor ownership and returns at mos No TTL, pruning job, worker update/delete port or cascading parent deletion. Retain operations, observations and continuation history for the Pack lifetime, including after settlement/retry. Parent deletion is RESTRICT; any owner-data deletion requires a separately authorized administrative process. Growth is bounded per approved paid attempt, not falsely bounded per owner lifetime. Access revocation hides retained history without erasing it. Actual proposal tests register operations and continuation grants through the real fenced command, append via a separate service-role transaction, and then deliver an actual refused stale write. They check unchanged Pack/wallet/usage snapshots, bounded concurrent conflicts, owner/claim mismatch, pagination, redaction, privilege denial and same-attempt continuation. The paired API integration uses the same proposal files. These are disposable local PostgreSQL checks, not seeded-history-only mechanics, hosted qualification or live provider evidence. + +Append accepts READ COMMITTED transactions only; higher isolation levels are refused before journal mutation. Pagination uses the owner/job/operation-ID index. Accepted-slot conflicts hash the original accepted payload, including after completion. diff --git a/proposals/pack-worker-fence-2269/command.sql b/proposals/pack-worker-fence-2269/command.sql index d76cd0e..3d286ed 100644 --- a/proposals/pack-worker-fence-2269/command.sql +++ b/proposals/pack-worker-fence-2269/command.sql @@ -9,6 +9,7 @@ DECLARE owner uuid; j public.release_pack_jobs; a public.release_pack_assets; phase text; wanted_model text; wanted_credits bigint; continuation uuid[]; supplied jsonb:=p_input; BEGIN owner:=public.authorize_pack_actor(p_actor,p_org); + IF p_action IS NULL OR p_action NOT IN('prepare','estimate','export','read','approve','resume','run','dispatch_uncertain','claim','context','phase','finish_asset','settle') THEN RAISE EXCEPTION 'Unknown protocol-2 command'; END IF; IF jsonb_typeof(p_input) IS DISTINCT FROM 'object' THEN RAISE EXCEPTION 'Invalid Pack input'; END IF; IF p_action='prepare' AND NOT(p_input ? 'expected_revision') THEN -- Shared with legacy prepare. Protocol ownership is set on INSERT, never promoted. @@ -70,7 +71,7 @@ BEGIN IF p_action='run' THEN IF nullif(p_input->>'workflow_run_id','') IS NULL OR length(p_input->>'workflow_run_id')>200 THEN RAISE EXCEPTION 'Invalid workflow identity'; END IF; IF e.workflow_run_id IS NOT NULL AND e.workflow_run_id<>p_input->>'workflow_run_id' THEN RAISE EXCEPTION 'Workflow identity conflict' USING ERRCODE='P0003'; END IF; - ELSIF p_action<>'dispatch_uncertain' THEN + ELSIF p_action<>'dispatch_uncertain' OR e.workflow_run_id IS NOT NULL THEN IF e.workflow_run_id IS NULL OR p_input->>'workflow_run_id' IS DISTINCT FROM e.workflow_run_id THEN RAISE EXCEPTION 'Stale workflow' USING ERRCODE='P0003'; END IF; END IF; IF p_action IN('phase','finish_asset') THEN @@ -87,16 +88,10 @@ BEGIN IF e.settlement_payload<>p_input THEN RAISE EXCEPTION 'Settlement payload conflict'; END IF; RETURN e.settlement_result; END IF; - IF j.status<>'running' THEN RAISE EXCEPTION 'Pack not running'; END IF; - IF j.hold_expires_at>'workflow_run_id' WHERE id=e.id AND workflow_run_id IS NULL; - ELSIF p_action='dispatch_uncertain' THEN - IF e.workflow_run_id IS NOT NULL THEN RETURN public.release_pack_command_legacy_internal(p_actor,p_org,'read',j.id,'{}'); END IF; + -- Action-specific CAS and read-only replay decisions are part of the fence too. + -- Do not let an expired hold turn stale input or an old claim replay into settlement. + IF p_action='context' THEN + IF (p_input->>'context_version')::integer IS DISTINCT FROM j.context_version THEN RAISE EXCEPTION 'Context version changed'; END IF; ELSIF p_action='claim' THEN request:=(p_input->>'request_id')::uuid; IF request IS NULL THEN RAISE EXCEPTION 'Claim request identity required'; END IF; @@ -112,6 +107,23 @@ BEGIN IF (p_input->>'asset_attempt')::integer IS DISTINCT FROM a.attempt THEN RAISE EXCEPTION 'Stale pending attempt' USING ERRCODE='P0003'; END IF; IF e.resume_asset_id=a.id THEN IF a.status<>'running' THEN RAISE EXCEPTION 'Resume asset not running'; END IF; + ELSIF a.status<>'pending' THEN + RETURN jsonb_build_object('job',to_jsonb(j),'asset',to_jsonb(a),'claimed',false); + END IF; + END IF; + IF j.status<>'running' THEN RAISE EXCEPTION 'Pack not running'; END IF; + IF j.hold_expires_at>'workflow_run_id' WHERE id=e.id AND workflow_run_id IS NULL; + ELSIF p_action='dispatch_uncertain' THEN + IF e.workflow_run_id IS NOT NULL THEN RETURN public.release_pack_command_legacy_internal(p_actor,p_org,'read',j.id,'{}'); END IF; + ELSIF p_action='claim' THEN + IF e.resume_asset_id=a.id THEN SELECT coalesce(array_agg(id),'{}') INTO continuation FROM public.release_pack_provider_operations WHERE owner_id=owner AND asset_id=a.id AND attempt=a.attempt; ELSE result:=public.release_pack_command_legacy_internal(p_actor,p_org,'claim',j.id,p_input); @@ -126,7 +138,6 @@ BEGIN UPDATE public.release_pack_claims SET claim_result=result WHERE id=c.id; RETURN result; ELSIF p_action='context' THEN - IF (p_input->>'context_version')::integer IS DISTINCT FROM j.context_version THEN RAISE EXCEPTION 'Context version changed'; END IF; UPDATE public.release_pack_jobs SET context_version=context_version+1 WHERE id=j.id; ELSIF p_action='phase' THEN phase:=p_input->>'phase'; value:=p_input->'value'; previous:=a.provider_state->phase; @@ -153,6 +164,7 @@ BEGIN wanted_model:=CASE WHEN has_foundation OR coalesce(jsonb_array_length(j.quote->'sources'->'references'),jsonb_array_length(j.brief->'reference_image_urls'),0)>0 THEN j.quote->'execution_config'->'models'->>'image_edit' ELSE j.quote->'execution_config'->'models'->>'image' END; wanted_credits:=greatest(1,round((j.quote->'execution_config'->'rates'->>'image_usd_per_unit')::numeric*(j.quote->'execution_config'->'rates'->>'credits_per_usd')::numeric)); ELSE RAISE EXCEPTION 'Provider phase outside quoted plan'; END IF; + IF value->>'state'='submitting' AND value ? 'request_id' THEN RAISE EXCEPTION 'Submitting intent cannot contain provider receipt'; END IF; SELECT o.* INTO op FROM public.release_pack_provider_operations o WHERE o.owner_id=owner AND o.asset_id=a.id AND o.attempt=a.attempt AND o.phase=cmd.phase; IF NOT FOUND THEN IF value->>'state' IS DISTINCT FROM 'submitting' OR wanted_model IS NULL OR value->>'model' IS DISTINCT FROM wanted_model @@ -173,7 +185,7 @@ BEGIN p_input:=p_input||jsonb_build_object('value',value); ELSIF previous IS NOT NULL AND previous<>value THEN RAISE EXCEPTION 'Saved phase conflict'; END IF; ELSIF p_action='finish_asset' THEN - IF p_input->>'status'='failed' AND EXISTS(SELECT 1 FROM jsonb_each(a.provider_state) entry WHERE entry.key LIKE '%submission%' AND entry.value->>'state' IS DISTINCT FROM 'complete') THEN + IF p_input->>'status' IN('failed','completed') AND EXISTS(SELECT 1 FROM jsonb_each(a.provider_state) entry WHERE entry.key LIKE '%submission%' AND entry.value->>'state' IS DISTINCT FROM 'complete') THEN RAISE EXCEPTION 'Incomplete provider attempt requires uncertainty'; END IF; ELSIF p_action<>'settle' THEN RAISE EXCEPTION 'Unknown protocol-2 command'; diff --git a/proposals/pack-worker-fence-2269/receipts.sql b/proposals/pack-worker-fence-2269/receipts.sql index 072afb0..e49e312 100644 --- a/proposals/pack-worker-fence-2269/receipts.sql +++ b/proposals/pack-worker-fence-2269/receipts.sql @@ -16,6 +16,7 @@ LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog AS $$ DECLARE op public.release_pack_provider_operations; effective_owner uuid; k text; existing jsonb; accepted jsonb; terminal jsonb; conflict jsonb; previous_conflict jsonb; BEGIN + IF current_setting('transaction_isolation')<>'read committed' THEN RAISE EXCEPTION 'Receipt append requires READ COMMITTED'; END IF; effective_owner:=public.authorize_pack_actor(p_actor,p_org); SELECT * INTO STRICT op FROM public.release_pack_provider_operations WHERE id=p_operation AND owner_id=effective_owner; IF NOT (((p_identity->>'execution_id')::uuid=op.execution_id AND (p_identity->>'claim_id')::uuid=op.claim_id) OR EXISTS( diff --git a/proposals/pack-worker-fence-2269/schema.sql b/proposals/pack-worker-fence-2269/schema.sql index 24bfb61..4202e77 100644 --- a/proposals/pack-worker-fence-2269/schema.sql +++ b/proposals/pack-worker-fence-2269/schema.sql @@ -33,6 +33,7 @@ CREATE TABLE public.release_pack_provider_operations ( charge_credits bigint NOT NULL CHECK(charge_credits>0), created_at timestamptz NOT NULL DEFAULT now(), UNIQUE(owner_id,asset_id,attempt,phase) ); +CREATE INDEX release_pack_provider_operations_page ON public.release_pack_provider_operations(owner_id,job_id,id); CREATE TABLE public.release_pack_provider_observations ( operation_id uuid NOT NULL REFERENCES public.release_pack_provider_operations(id) ON DELETE RESTRICT, kind text NOT NULL CHECK(kind IN('accepted','completed','failed','conflict')), diff --git a/proposals/pack-worker-fence-2269/test_expiry_fence.py b/proposals/pack-worker-fence-2269/test_expiry_fence.py new file mode 100644 index 0000000..10e7877 --- /dev/null +++ b/proposals/pack-worker-fence-2269/test_expiry_fence.py @@ -0,0 +1,79 @@ +"""Action-specific CAS/replay must be checked before expiry may mutate state.""" +import unittest +import uuid +from test_contract import FencedPacks + + +class ExpiryFence(FencedPacks): + def expire(self): + self.sql(f"UPDATE release_pack_jobs SET hold_expires_at=now()-interval '1 second' WHERE id='{self.job}'") + + def complete_state(self): + history = self.sql(f"SELECT jsonb_build_object('executions',(SELECT jsonb_agg(to_jsonb(e) ORDER BY id) FROM release_pack_executions e WHERE job_id='{self.job}'),'claims',(SELECT jsonb_agg(to_jsonb(c) ORDER BY id) FROM release_pack_claims c WHERE owner_id='{self.owner}'))").stdout.strip() + return self.snapshot(), history + + def test_context_cas_refuses_before_expiry_and_current_version_can_recover(self): + fence = self.running() + self.command('context', {**fence, 'context_version': 0, 'value': {'state': 'complete'}}) + self.expire() + before = self.complete_state() + for extra in [{'context_version': 0}, {'context_version': None}, {}]: + rejected = self.command('context', {**fence, **extra, 'value': {'stale': True}}, succeeds=False) + self.assertNotEqual(rejected.returncode, 0, rejected.stdout) + self.assertIn('Context version changed', rejected.stderr) + self.assertEqual(before, self.complete_state()) + recovered = self.command('context', {**fence, 'context_version': 1, 'value': {'not_saved_after_expiry': True}}) + self.assertTrue(recovered['command_expired']) + self.assertEqual(recovered['job']['context_version'], 1) + self.assertEqual(recovered['job']['production_context'], {'state': 'complete'}) + self.assertEqual(recovered['job']['held_credits'], 0) + after = self.complete_state() + self.command('read') + self.assertEqual(after, self.complete_state()) + + def test_pending_claim_cas_and_request_identity_refuse_before_expiry(self): + saved, _ = self.approve_v2() + fence = self.bind(saved) + asset_id = saved['assets'][0]['id'] + self.expire() + before = self.complete_state() + for extra, message in [ + ({'asset_attempt': 99, 'request_id': str(uuid.uuid4())}, 'Stale pending attempt'), + ({'request_id': str(uuid.uuid4())}, 'Stale pending attempt'), + ({'asset_attempt': 0}, 'Claim request identity required'), + ]: + rejected = self.command('claim', {**fence, 'asset_id': asset_id, **extra}, succeeds=False) + self.assertNotEqual(rejected.returncode, 0, rejected.stdout) + self.assertIn(message, rejected.stderr) + self.assertEqual(before, self.complete_state()) + expired = self.command('claim', {**fence, 'asset_id': asset_id, 'asset_attempt': 0, 'request_id': str(uuid.uuid4())}) + self.assertTrue(expired['command_expired']) + self.assertEqual(expired['job']['held_credits'], 0) + self.assertEqual(self.sql(f"SELECT count(*) FROM release_pack_claims WHERE owner_id='{self.owner}'").stdout.strip(), '0') + after = self.complete_state() + self.command('read') + self.assertEqual(after, self.complete_state()) + + def test_claim_replay_or_conflict_never_becomes_expiry_authority(self): + saved, _ = self.approve_v2() + fence = self.bind(saved) + payload = {**fence, 'asset_id': saved['assets'][0]['id'], 'asset_attempt': 0, 'request_id': str(uuid.uuid4())} + self.command('claim', payload) + self.expire() + before = self.complete_state() + replay = self.command('claim', payload) + self.assertTrue(replay.get('replayed'), replay) + self.assertFalse(replay['claimed']) + self.assertEqual(before, self.complete_state()) + conflict = self.command('claim', {**payload, 'asset_attempt': 1}, succeeds=False) + self.assertNotEqual(conflict.returncode, 0, conflict.stdout) + self.assertIn('Claim payload conflict', conflict.stderr) + self.assertEqual(before, self.complete_state()) + loser = self.command('claim', {**payload, 'request_id': str(uuid.uuid4())}) + self.assertFalse(loser['claimed']) + self.assertEqual(before, self.complete_state()) + + +if __name__ == '__main__': + suite = unittest.TestSuite(ExpiryFence(name) for name in ExpiryFence.__dict__ if name.startswith('test_')) + raise SystemExit(not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful()) diff --git a/proposals/pack-worker-fence-2269/test_extended.py b/proposals/pack-worker-fence-2269/test_extended.py index 8f20d6a..e8b1aed 100644 --- a/proposals/pack-worker-fence-2269/test_extended.py +++ b/proposals/pack-worker-fence-2269/test_extended.py @@ -5,6 +5,8 @@ import uuid from test_contract import FencedPacks, ReleasePackJobs from test_creation_race import CreationRace +from test_expiry_fence import ExpiryFence +from test_review_regressions import ReviewRegressions class Extended(FencedPacks): def register(self,f,phase='image_submission_0'): @@ -102,7 +104,7 @@ def setUpClass(cls): if __name__=='__main__': suite=unittest.TestSuite() - for cls in [FencedPacks,Extended,CreationRace]: + for cls in [FencedPacks,Extended,CreationRace,ExpiryFence,ReviewRegressions]: suite.addTests(cls(x) for x in cls.__dict__ if x.startswith('test_')) suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(LegacyCompatibility)) result=unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/proposals/pack-worker-fence-2269/test_review_regressions.py b/proposals/pack-worker-fence-2269/test_review_regressions.py new file mode 100644 index 0000000..e9832f3 --- /dev/null +++ b/proposals/pack-worker-fence-2269/test_review_regressions.py @@ -0,0 +1,96 @@ +"""Post-review boundary regressions against actual disposable SQL.""" +import hashlib +import json +import unittest +from test_contract import FencedPacks + +class ReviewRegressions(FencedPacks): + def register(self, fence, **extra): + value = {'state': 'submitting', 'model': 'fixture', 'input_sha256': 'a'*64, 'charge_credits': 40, **extra} + return self.command('phase', {**fence, 'phase': 'image_submission_0', 'value': value}), value + + def test_completed_asset_requires_complete_provider_journal(self): + f = self.running() + registered, value = self.register(f) + op = registered['operation_id'] + for state in ['submitting', 'accepted']: + if state == 'accepted': + self.observe(op, f, {'kind': 'accepted', 'request_id': 'original'}) + self.command('phase', {**f, 'phase': 'image_submission_0', 'value': {**value, 'state': state, 'request_id': 'original'}}) + before = self.snapshot() + rejected = self.command('finish_asset', {**f, 'status': 'completed', 'files': [{'storage_key': 'fixture'}], 'caption': 'fixture', 'qa': {'passed': True}}, succeeds=False) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn('Incomplete provider attempt', rejected.stderr) + self.assertEqual(before, self.snapshot()) + self.observe(op, f, {'kind': 'completed', 'request_id': 'original'}) + self.command('phase', {**f, 'phase': 'image_submission_0', 'value': {**value, 'state': 'complete', 'request_id': 'original'}}) + self.command('finish_asset', {**f, 'status': 'completed', 'files': [{'storage_key': 'fixture'}], 'caption': 'fixture', 'qa': {'passed': True}}) + + def test_submitting_intent_cannot_preseed_request_id_or_skip_charge(self): + f = self.running() + before = self.snapshot() + for request_id in ['original', None]: + value = {'state': 'submitting', 'model': 'fixture', 'input_sha256': 'a'*64, 'charge_credits': 40, 'request_id': request_id} + rejected = self.command('phase', {**f, 'phase': 'image_submission_0', 'value': value}, succeeds=False) + self.assertNotEqual(rejected.returncode, 0) + self.assertEqual(before, self.snapshot()) + registered, value = self.register(f) + self.observe(registered['operation_id'], f, {'kind': 'accepted', 'request_id': 'original'}) + result = self.command('phase', {**f, 'phase': 'image_submission_0', 'value': {**value, 'state': 'accepted', 'request_id': 'original'}}) + self.assertEqual(self.sql(f"SELECT billed_credits FROM release_pack_assets WHERE id='{f['asset_id']}'").stdout.strip(), '40') + self.assertEqual(self.sql(f"SELECT count(*) FROM usage_events WHERE account_id='{self.owner}'").stdout.strip(), '1') + + def test_invalid_actions_refuse_before_expiry(self): + f = self.running() + self.sql(f"UPDATE release_pack_jobs SET hold_expires_at=now()-interval '1 second' WHERE id='{self.job}'") + before = self.snapshot() + for action in ["'unknown'", 'NULL']: + payload = json.dumps(f) + result = self.sql(f"SET ROLE service_role; SELECT release_pack_command_v2('{self.owner}',null,{action},'{self.job}','{payload}')", succeeds=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Unknown protocol-2 command', result.stderr) + self.assertEqual(before, self.snapshot()) + + def test_expired_settlement_replays_identical_response(self): + f = self.running() + self.sql(f"UPDATE release_pack_jobs SET hold_expires_at=now()-interval '1 second' WHERE id='{self.job}'") + result = self.command('settle', f) + before = self.snapshot() + self.assertTrue(result['command_expired']) + self.assertEqual(self.command('settle', f), result) + self.assertEqual(before, self.snapshot()) + self.assertNotEqual(self.command('settle', {**f, 'error': 'changed'}, succeeds=False).returncode, 0) + + def test_bound_dispatch_uncertain_requires_workflow_identity_before_expiry(self): + f = self.running() + self.sql(f"UPDATE release_pack_jobs SET hold_expires_at=now()-interval '1 second' WHERE id='{self.job}'") + before = self.snapshot() + for run in [None, 'wrong']: + rejected = self.command('dispatch_uncertain', {**f, 'workflow_run_id': run}, succeeds=False) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn('Stale workflow', rejected.stderr) + self.assertEqual(before, self.snapshot()) + + def test_receipts_reject_unsupported_transaction_isolation(self): + f = self.running() + registered, _ = self.register(f) + identity = json.dumps({'execution_id': f['execution_id'], 'claim_id': f['claim_id']}) + for isolation in ['REPEATABLE READ', 'SERIALIZABLE']: + result = self.sql(f"BEGIN ISOLATION LEVEL {isolation}; SET ROLE service_role; SELECT append_pack_provider_observation('{self.owner}',null,'{registered['operation_id']}','{identity}','{{\"kind\":\"accepted\",\"request_id\":\"original\"}}'); COMMIT", succeeds=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn('requires READ COMMITTED', result.stderr) + self.assertEqual(self.observe(registered['operation_id'], f, {'kind': 'accepted', 'request_id': 'original'})['state'], 'saved') + + def test_accepted_conflict_hashes_original_accepted_slot(self): + f = self.running() + registered, _ = self.register(f) + op = registered['operation_id'] + self.observe(op, f, {'kind': 'accepted', 'request_id': 'original'}) + self.observe(op, f, {'kind': 'completed', 'request_id': 'original'}) + self.observe(op, f, {'kind': 'accepted', 'request_id': 'different'}) + hashes = self.sql(f"SELECT (SELECT payload->>'existing_sha256' FROM release_pack_provider_observations WHERE operation_id='{op}' AND kind='conflict')=encode(sha256(convert_to(payload::text,'UTF8')),'hex') FROM release_pack_provider_observations WHERE operation_id='{op}' AND kind='accepted'").stdout.strip() + self.assertEqual(hashes, 't') + +if __name__ == '__main__': + suite = unittest.TestSuite(ReviewRegressions(name) for name in ReviewRegressions.__dict__ if name.startswith('test_')) + raise SystemExit(not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful())