diff --git a/.github/workflows/credit-reservations.yml b/.github/workflows/credit-reservations.yml new file mode 100644 index 0000000..efae730 --- /dev/null +++ b/.github/workflows/credit-reservations.yml @@ -0,0 +1,33 @@ +name: Credit reservation transactions + +on: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +jobs: + credit-reservations: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + postgres: [15, 16] + steps: + - uses: actions/checkout@v4 + - name: Install PostgreSQL + run: | + sudo apt-get update + sudo apt-get install -y postgresql-common + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get update + sudo apt-get install -y postgresql-${{ matrix.postgres }} + - name: Test isolated credit reservations + run: | + export PATH="/usr/lib/postgresql/${{ matrix.postgres }}/bin:$PATH" + python3 -m unittest discover -s tests -p 'test_credit_reservations.py' + python3 -m unittest discover -s tests -p 'test_credit_reservation_composition.py' diff --git a/supabase/migrations/20261010210500_credit_reservations.sql b/supabase/migrations/20261010210500_credit_reservations.sql new file mode 100644 index 0000000..cd5b6fd --- /dev/null +++ b/supabase/migrations/20261010210500_credit_reservations.sql @@ -0,0 +1,200 @@ +-- Shared credit reservations (holds) on the existing wallet for AI work +-- (recoupable/app#2105 milestone 1, reused by Context accounting app#2123). +-- +-- Wallet invariant: confirmed balance = credits_usage.remaining_credits; +-- outstanding holds = sum(credits_held) of this owner's 'held' reservations; +-- spendable = confirmed balance - outstanding holds. A hold is a claim, never a +-- temporary debit, so refill, top-up and auto-top-up logic never see it as use. +-- Settlement charges through record_credit_charge_once with the same operation +-- key, so a reservation and its charge share one retry-stable identity. +-- +-- Not a price, spending ceiling, top-up, expiry policy, or activation of any +-- caller. Legacy debit writers do not consult holds yet. Holds never expire on +-- their own: unknown provider exposure stays held until explicitly reconciled. +BEGIN; + +CREATE TABLE public.credit_reservations ( + id text PRIMARY KEY CHECK (id ~ '^hold-v1-[a-f0-9]{64}$'), + account_id uuid NOT NULL, + operation_key text NOT NULL + CHECK (btrim(operation_key) <> '' AND length(operation_key) <= 200), + credits_held bigint NOT NULL CHECK (credits_held > 0 AND credits_held <= 9007199254740991), + state text NOT NULL DEFAULT 'held' CHECK (state IN ('held','settled','released')), + credits_charged bigint CHECK (credits_charged > 0 AND credits_charged <= credits_held), + charge_event_id text CHECK (charge_event_id ~ '^charge-v1-[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT now(), + closed_at timestamptz, + UNIQUE (account_id, operation_key), + CHECK ((state = 'held' AND credits_charged IS NULL AND charge_event_id IS NULL AND closed_at IS NULL) + OR (state = 'settled' AND credits_charged IS NOT NULL AND charge_event_id IS NOT NULL + AND closed_at IS NOT NULL) + OR (state = 'released' AND credits_charged IS NULL AND charge_event_id IS NULL + AND closed_at IS NOT NULL)) +); +CREATE INDEX credit_reservations_open_idx + ON public.credit_reservations (account_id) WHERE state = 'held'; +ALTER TABLE public.credit_reservations ENABLE ROW LEVEL SECURITY; +-- Retained like charge receipts: deleting a row would let a recycled key hold again, +-- so the service role gets no DELETE/TRUNCATE even where default privileges grant it. +-- UPDATE is limited to the columns settle/release close; a hold's owner, key and +-- amount are immutable once written. The service role may read open holds +-- (state = 'held', by account and created_at) to reconcile lost operation keys. +REVOKE ALL ON public.credit_reservations FROM PUBLIC, anon, authenticated, service_role; +GRANT SELECT, INSERT ON public.credit_reservations TO service_role; +GRANT UPDATE (state, credits_charged, charge_event_id, closed_at) + ON public.credit_reservations TO service_role; + +CREATE OR REPLACE FUNCTION public.reserve_credits_once( + p_account_id uuid, p_operation_key text, p_credits bigint +) RETURNS jsonb +LANGUAGE plpgsql SECURITY INVOKER SET search_path = public, pg_temp AS $$ +DECLARE + v_id text; + v_held bigint; + v_hold public.credit_reservations%ROWTYPE; + v_wallet public.credits_usage%ROWTYPE; +BEGIN + IF p_account_id IS NULL OR p_operation_key IS NULL OR btrim(p_operation_key) = '' + OR length(p_operation_key) > 200 OR p_credits IS NULL OR p_credits <= 0 + OR p_credits > 9007199254740991 THEN + RAISE EXCEPTION 'Invalid credit reservation' USING ERRCODE = '22023'; + END IF; + -- Every reservation operation locks the owner's single wallet row first, so + -- holds serialize with each other, charge receipts and legacy wallet writers. + BEGIN + SELECT * INTO STRICT v_wallet FROM public.credits_usage + WHERE account_id = p_account_id ORDER BY id FOR UPDATE; + EXCEPTION WHEN no_data_found OR too_many_rows THEN + RAISE EXCEPTION 'Credit wallet is unavailable or ambiguous' USING ERRCODE = '22023'; + END; + IF v_wallet.remaining_credits IS NULL THEN + RAISE EXCEPTION 'Credit wallet is unavailable' USING ERRCODE = '22023'; + END IF; + v_id := 'hold-v1-' || encode(sha256(convert_to( + jsonb_build_array(p_account_id::text,p_operation_key)::text,'UTF8')),'hex'); + SELECT * INTO v_hold FROM public.credit_reservations WHERE id = v_id; + IF FOUND THEN + IF v_hold.account_id IS DISTINCT FROM p_account_id + OR v_hold.operation_key IS DISTINCT FROM p_operation_key + OR v_hold.credits_held IS DISTINCT FROM p_credits THEN + RAISE EXCEPTION 'Credit reservation identity conflict' USING ERRCODE = '22023'; + END IF; + -- A replay reports the saved status; it never re-holds released or settled work. + RETURN jsonb_build_object('state','reused','reservationId',v_id, + 'status',v_hold.state,'creditsHeld',v_hold.credits_held); + END IF; + SELECT coalesce(sum(credits_held),0)::bigint INTO v_held FROM public.credit_reservations + WHERE account_id = p_account_id AND state = 'held'; + IF v_wallet.remaining_credits - v_held < p_credits THEN + -- A definite denial writes nothing; the same key may be retried later. + RETURN jsonb_build_object('state','insufficient','creditsRequested',p_credits, + 'spendableCredits',v_wallet.remaining_credits - v_held); + END IF; + INSERT INTO public.credit_reservations(id,account_id,operation_key,credits_held) + VALUES (v_id,p_account_id,p_operation_key,p_credits); + RETURN jsonb_build_object('state','held','reservationId',v_id,'creditsHeld',p_credits, + 'spendableCredits',v_wallet.remaining_credits - v_held - p_credits); +END; +$$; + +CREATE OR REPLACE FUNCTION public.settle_credit_reservation( + p_account_id uuid, p_operation_key text, p_credits bigint, p_event jsonb +) RETURNS jsonb +LANGUAGE plpgsql SECURITY INVOKER SET search_path = public, pg_temp AS $$ +DECLARE + v_id text; + v_charge jsonb; + v_hold public.credit_reservations%ROWTYPE; + v_wallet public.credits_usage%ROWTYPE; +BEGIN + IF p_account_id IS NULL OR p_operation_key IS NULL OR btrim(p_operation_key) = '' + OR length(p_operation_key) > 200 OR p_credits IS NULL OR p_credits <= 0 + OR p_credits > 9007199254740991 THEN + RAISE EXCEPTION 'Invalid credit settlement' USING ERRCODE = '22023'; + END IF; + BEGIN + SELECT * INTO STRICT v_wallet FROM public.credits_usage + WHERE account_id = p_account_id ORDER BY id FOR UPDATE; + EXCEPTION WHEN no_data_found OR too_many_rows THEN + RAISE EXCEPTION 'Credit wallet is unavailable or ambiguous' USING ERRCODE = '22023'; + END; + v_id := 'hold-v1-' || encode(sha256(convert_to( + jsonb_build_array(p_account_id::text,p_operation_key)::text,'UTF8')),'hex'); + SELECT * INTO v_hold FROM public.credit_reservations WHERE id = v_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'Credit reservation not found' USING ERRCODE = '22023'; + END IF; + IF v_hold.state = 'released' THEN + -- Charging released work needs an explicit failure/cancellation policy. + RAISE EXCEPTION 'Credit reservation was released' USING ERRCODE = '22023'; + END IF; + IF p_credits > v_hold.credits_held THEN + RAISE EXCEPTION 'Credit settlement exceeds reservation' USING ERRCODE = '22023'; + END IF; + -- Same owner/key as the reservation: an identical retry reuses the charge + -- receipt; a changed amount or usage payload raises its identity conflict. + v_charge := public.record_credit_charge_once(p_account_id,p_operation_key,p_credits,p_event); + IF v_hold.state = 'held' THEN + UPDATE public.credit_reservations + SET state = 'settled', credits_charged = p_credits, + charge_event_id = v_charge->>'eventId', closed_at = now() + WHERE id = v_id; + ELSIF v_hold.charge_event_id IS DISTINCT FROM v_charge->>'eventId' + OR v_hold.credits_charged IS DISTINCT FROM p_credits THEN + RAISE EXCEPTION 'Credit settlement identity conflict' USING ERRCODE = '22023'; + END IF; + RETURN jsonb_build_object( + 'state',CASE WHEN v_hold.state = 'held' THEN 'settled' ELSE 'reused' END, + 'reservationId',v_id,'eventId',v_charge->>'eventId','creditsHeld',v_hold.credits_held, + 'creditsCharged',p_credits,'creditsReleased',v_hold.credits_held - p_credits); +END; +$$; + +CREATE OR REPLACE FUNCTION public.release_credit_reservation( + p_account_id uuid, p_operation_key text +) RETURNS jsonb +LANGUAGE plpgsql SECURITY INVOKER SET search_path = public, pg_temp AS $$ +DECLARE + v_id text; + v_hold public.credit_reservations%ROWTYPE; + v_wallet public.credits_usage%ROWTYPE; +BEGIN + IF p_account_id IS NULL OR p_operation_key IS NULL OR btrim(p_operation_key) = '' + OR length(p_operation_key) > 200 THEN + RAISE EXCEPTION 'Invalid credit release' USING ERRCODE = '22023'; + END IF; + BEGIN + SELECT * INTO STRICT v_wallet FROM public.credits_usage + WHERE account_id = p_account_id ORDER BY id FOR UPDATE; + EXCEPTION WHEN no_data_found OR too_many_rows THEN + RAISE EXCEPTION 'Credit wallet is unavailable or ambiguous' USING ERRCODE = '22023'; + END; + v_id := 'hold-v1-' || encode(sha256(convert_to( + jsonb_build_array(p_account_id::text,p_operation_key)::text,'UTF8')),'hex'); + SELECT * INTO v_hold FROM public.credit_reservations WHERE id = v_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'Credit reservation not found' USING ERRCODE = '22023'; + END IF; + IF v_hold.state = 'settled' THEN + RAISE EXCEPTION 'Credit reservation was settled' USING ERRCODE = '22023'; + END IF; + -- Release writes no usage event and never changes the confirmed balance. + IF v_hold.state = 'held' THEN + UPDATE public.credit_reservations SET state = 'released', closed_at = now() WHERE id = v_id; + END IF; + RETURN jsonb_build_object( + 'state',CASE WHEN v_hold.state = 'held' THEN 'released' ELSE 'reused' END, + 'reservationId',v_id,'creditsHeld',v_hold.credits_held,'creditsReleased',v_hold.credits_held); +END; +$$; + +REVOKE ALL ON FUNCTION public.reserve_credits_once(uuid,text,bigint) + FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION public.reserve_credits_once(uuid,text,bigint) TO service_role; +REVOKE ALL ON FUNCTION public.settle_credit_reservation(uuid,text,bigint,jsonb) + FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION public.settle_credit_reservation(uuid,text,bigint,jsonb) TO service_role; +REVOKE ALL ON FUNCTION public.release_credit_reservation(uuid,text) + FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION public.release_credit_reservation(uuid,text) TO service_role; +COMMIT; diff --git a/supabase/migrations/20261117000000_compose_credit_reservations.sql b/supabase/migrations/20261117000000_compose_credit_reservations.sql new file mode 100644 index 0000000..d86efd1 --- /dev/null +++ b/supabase/migrations/20261117000000_compose_credit_reservations.sql @@ -0,0 +1,157 @@ +-- Compose the existing shared-reservation candidate with released pack holds and +-- all wallet debit writers. Depends on 20261010210500_credit_reservations and +-- 20261020150000_release_pack_jobs. No balances, prices or callers are changed. +-- Pack holds have already been deducted from remaining_credits. Shared holds +-- have not: protect only the latter at the common wallet-write boundary. +BEGIN; +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '30s'; + +CREATE OR REPLACE FUNCTION public.protect_credit_reservations() +RETURNS trigger LANGUAGE plpgsql SECURITY DEFINER +SET search_path = public, pg_temp AS $$ +DECLARE v_held numeric; +BEGIN + -- Definer visibility is necessary: a permitted legacy writer may not be + -- allowed to read reservation rows. This function grants no read/write API. + SELECT coalesce(sum(credits_held),0) INTO v_held + FROM public.credit_reservations + WHERE account_id = OLD.account_id AND state = 'held'; + IF v_held > 0 THEN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'Outstanding credit reservations prevent wallet removal' + USING ERRCODE = '22023'; + END IF; + IF NEW.account_id IS DISTINCT FROM OLD.account_id + OR NEW.remaining_credits IS NULL OR NEW.remaining_credits < v_held THEN + RAISE EXCEPTION 'Outstanding credit reservations exceed available wallet funds' + USING ERRCODE = '22023'; + END IF; + END IF; + IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; + RETURN NEW; +END; +$$; +REVOKE ALL ON FUNCTION public.protect_credit_reservations() + FROM PUBLIC, anon, authenticated, service_role; +CREATE TRIGGER protect_credit_reservations +BEFORE UPDATE OF remaining_credits, account_id OR DELETE ON public.credits_usage +FOR EACH ROW EXECUTE FUNCTION public.protect_credit_reservations(); + +CREATE OR REPLACE FUNCTION public.reserve_credits_once( + p_account_id uuid, p_operation_key text, p_credits bigint +) RETURNS jsonb +LANGUAGE plpgsql SECURITY INVOKER SET search_path = public, pg_temp AS $$ +DECLARE + v_id text; + v_held bigint; + v_hold public.credit_reservations%ROWTYPE; + v_wallet public.credits_usage%ROWTYPE; +BEGIN + IF p_account_id IS NULL OR p_operation_key IS NULL OR btrim(p_operation_key) = '' + OR length(p_operation_key) > 200 OR p_credits IS NULL OR p_credits <= 0 + OR p_credits > 9007199254740991 THEN + RAISE EXCEPTION 'Invalid credit reservation' USING ERRCODE = '22023'; + END IF; + -- Every reservation operation locks the owner's single wallet row first, so + -- holds serialize with each other, charge receipts and legacy wallet writers. + BEGIN + SELECT * INTO STRICT v_wallet FROM public.credits_usage + WHERE account_id = p_account_id ORDER BY id FOR UPDATE; + EXCEPTION WHEN no_data_found OR too_many_rows THEN + RAISE EXCEPTION 'Credit wallet is unavailable or ambiguous' USING ERRCODE = '22023'; + END; + IF v_wallet.remaining_credits IS NULL THEN + RAISE EXCEPTION 'Credit wallet is unavailable' USING ERRCODE = '22023'; + END IF; + v_id := 'hold-v1-' || encode(sha256(convert_to( + jsonb_build_array(p_account_id::text,p_operation_key)::text,'UTF8')),'hex'); + SELECT * INTO v_hold FROM public.credit_reservations WHERE id = v_id; + IF FOUND THEN + IF v_hold.account_id IS DISTINCT FROM p_account_id + OR v_hold.operation_key IS DISTINCT FROM p_operation_key + OR v_hold.credits_held IS DISTINCT FROM p_credits THEN + RAISE EXCEPTION 'Credit reservation identity conflict' USING ERRCODE = '22023'; + END IF; + -- A replay reports the saved status; it never re-holds released or settled work. + RETURN jsonb_build_object('state','reused','reservationId',v_id, + 'status',v_hold.state,'creditsHeld',v_hold.credits_held); + END IF; + SELECT coalesce(sum(credits_held),0)::bigint INTO v_held FROM public.credit_reservations + WHERE account_id = p_account_id AND state = 'held'; + IF v_wallet.remaining_credits - v_held < p_credits THEN + -- A definite denial writes nothing; the same key may be retried later. + RETURN jsonb_build_object('state','insufficient','creditsRequested',p_credits, + 'spendableCredits',v_wallet.remaining_credits - v_held); + END IF; + INSERT INTO public.credit_reservations(id,account_id,operation_key,credits_held) + VALUES (v_id,p_account_id,p_operation_key,p_credits); + -- Version the locked wallet so an older REPEATABLE READ writer cannot miss + -- this new hold. This changes no amount, timestamp, usage event or top-up. + UPDATE public.credits_usage SET remaining_credits = remaining_credits + WHERE id = v_wallet.id; + RETURN jsonb_build_object('state','held','reservationId',v_id,'creditsHeld',p_credits, + 'spendableCredits',v_wallet.remaining_credits - v_held - p_credits); +END; +$$; + +CREATE OR REPLACE FUNCTION public.settle_credit_reservation( + p_account_id uuid, p_operation_key text, p_credits bigint, p_event jsonb +) RETURNS jsonb +LANGUAGE plpgsql SECURITY INVOKER SET search_path = public, pg_temp AS $$ +DECLARE + v_id text; + v_charge jsonb; + v_hold public.credit_reservations%ROWTYPE; + v_wallet public.credits_usage%ROWTYPE; +BEGIN + IF p_account_id IS NULL OR p_operation_key IS NULL OR btrim(p_operation_key) = '' + OR length(p_operation_key) > 200 OR p_credits IS NULL OR p_credits <= 0 + OR p_credits > 9007199254740991 THEN + RAISE EXCEPTION 'Invalid credit settlement' USING ERRCODE = '22023'; + END IF; + BEGIN + SELECT * INTO STRICT v_wallet FROM public.credits_usage + WHERE account_id = p_account_id ORDER BY id FOR UPDATE; + EXCEPTION WHEN no_data_found OR too_many_rows THEN + RAISE EXCEPTION 'Credit wallet is unavailable or ambiguous' USING ERRCODE = '22023'; + END; + v_id := 'hold-v1-' || encode(sha256(convert_to( + jsonb_build_array(p_account_id::text,p_operation_key)::text,'UTF8')),'hex'); + SELECT * INTO v_hold FROM public.credit_reservations WHERE id = v_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'Credit reservation not found' USING ERRCODE = '22023'; + END IF; + IF v_hold.state = 'released' THEN + -- Charging released work needs an explicit failure/cancellation policy. + RAISE EXCEPTION 'Credit reservation was released' USING ERRCODE = '22023'; + END IF; + IF p_credits > v_hold.credits_held THEN + RAISE EXCEPTION 'Credit settlement exceeds reservation' USING ERRCODE = '22023'; + END IF; + -- Same owner/key as the reservation: an identical retry reuses the charge + -- receipt; a changed amount or usage payload raises its identity conflict. + IF v_hold.state = 'held' THEN + -- Close only this hold before debiting, so the wallet guard still protects + -- every other reservation. Both writes are one transaction: invalid event, + -- identity conflict or any charge failure restores the original open hold. + -- Hold and charge IDs use the same owner/key digest, with different prefixes. + UPDATE public.credit_reservations + SET state = 'settled', credits_charged = p_credits, + charge_event_id = 'charge-v1-' || substr(v_id,9), closed_at = now() + WHERE id = v_id; + END IF; + v_charge := public.record_credit_charge_once(p_account_id,p_operation_key,p_credits,p_event); + IF v_hold.state = 'settled' AND + (v_hold.charge_event_id IS DISTINCT FROM v_charge->>'eventId' + OR v_hold.credits_charged IS DISTINCT FROM p_credits) THEN + RAISE EXCEPTION 'Credit settlement identity conflict' USING ERRCODE = '22023'; + END IF; + RETURN jsonb_build_object( + 'state',CASE WHEN v_hold.state = 'held' THEN 'settled' ELSE 'reused' END, + 'reservationId',v_id,'eventId',v_charge->>'eventId','creditsHeld',v_hold.credits_held, + 'creditsCharged',p_credits,'creditsReleased',v_hold.credits_held - p_credits); +END; +$$; + +COMMIT; diff --git a/tests/CREDIT-RESERVATIONS.md b/tests/CREDIT-RESERVATIONS.md new file mode 100644 index 0000000..efaa8eb --- /dev/null +++ b/tests/CREDIT-RESERVATIONS.md @@ -0,0 +1,118 @@ +# Credit reservations (holds) + +`20261010210500_credit_reservations.sql` adds the shared reserve/settle/release +primitive for AI work (recoupable/app#2105 milestone 1, reused by Context accounting +in recoupable/app#2123). It uses the existing `credits_usage` wallet and the +`usage_events` audit log through `record_credit_charge_once`. It adds no wallet, +balance column, price, ceiling, top-up, expiry or caller. + +## Wallet invariant + +- **Wallet balance:** `credits_usage.remaining_credits`, unchanged by shared holds. + Released pack holds have already been deducted from this balance. +- **Outstanding holds:** the sum of `credits_held` for the owner's `held` rows. +- **Spendable:** wallet balance minus outstanding shared holds. Never subtract pack holds twice. + +A hold is a claim, not a temporary debit followed by a refund, so the monthly +refill, top-ups and auto-top-up never treat it as consumption. Every operation locks +the owner's single wallet row first (rejecting zero or multiple rows like +`record_credit_charge_once`), so holds serialize with each other, with charge +receipts and with the legacy wallet writers. + +## Operations (service role only, security invoker) + +| Function | Result | +| --- | --- | +| `reserve_credits_once(owner, key, credits)` | `held` (new hold), `reused` (same key and amount; reports saved status), or `insufficient` (nothing written). A changed amount conflicts. | +| `settle_credit_reservation(owner, key, credits, event)` | Charges the actual credits through `record_credit_charge_once` with the same key and closes the hold; the remainder is no longer held. An identical retry is `reused`; a changed amount or usage payload conflicts. Charging more than the hold, or a released hold, is rejected. | +| `release_credit_reservation(owner, key)` | Closes an unsettled hold without a charge or usage event; a retry is `reused`. Releasing a settled hold is rejected. | + +The reservation ID (`hold-v1-…`) and the charge receipt ID (`charge-v1-…`) both +derive from the billing owner and operation key. A key identifies one chargeable +attempt: never generate a fresh key on retry and never reuse one for independent work. +A lost reply is ambiguous; replay the same call with the same key to learn the state. +Every rejection these functions raise uses SQLSTATE `22023` and rolls back, so it is +definite for that call: nothing was written, and an unchanged replay fails the same way. + +A `reused` receipt with status `held` goes to every duplicate delivery of the same +key, so a hold permits paid work but is not an execution lease. Duplicate deliveries +must be deduplicated by the caller's own attempt identity before provider dispatch; +settlement charges once either way. + +## Activation prerequisites + +This successor retains Database115's original migration unchanged and adds +`20261117000000_compose_credit_reservations.sql` after the released pack migration. +The added trigger checks all balance/owner updates and wallet deletion against open +shared holds. Unrelated metadata writes and legacy no-hold debit behavior are unchanged. +It uses a narrowly scoped SECURITY DEFINER function with pinned search path and no +direct EXECUTE grant, so a wallet writer cannot hide holds through its own RLS view. + +Settlement closes its own hold before the charge within one transaction; a failed +charge restores that hold. Other holds remain protected. Successful reserve performs +a no-amount-change wallet update: this creates a row version that forces an old +repeatable-read writer to serialize/retry instead of missing a new hold. + +Current released API refill/top-up/deduction callers are already atomic. The pack +migration's refill floor counts unconsumed pack holds. Preserve it; the old +Database101/API1009 prerequisite text is historical, not current implementation state. +Database88's one-wallet constraint remains separate; this migration does not apply it +or repair duplicates. Zero/multiple wallets still reject reservation operations. +No caller activation follows from these source changes. Review/release and verify +actual migrations/grants separately before any live use. + +## Deliberately open + +- **Legacy pre-dispatch authorization is not implemented.** The guard protects + reserved funds at debit time, but legacy callers can already have incurred provider + expense when a debit fails. It does not bound that exposure or activate paid work. +- **No expiry.** Unknown provider exposure stays held until the trusted server + settles or releases it. Do not add a time-based release that turns uncertain + work into spendable credit. +- **Failure and cancellation billing are not decided here.** Release is for work + known not to have incurred a charge (for example, cancelled before dispatch). + Charging released or over-limit work needs an approved policy. +- Request/pilot ceilings, provider-expense versus customer-debit reporting and + pricing versions are not stored here. +- Reservations must be retained like charge receipts. A purge would let a + recycled key hold capacity again, so the service role has no DELETE/TRUNCATE. + Its UPDATE grant covers only the closing columns (`state`, `credits_charged`, + `charge_event_id`, `closed_at`); owner, key and amount are immutable. +- **No listing API for open holds.** A worker that loses its key leaves the hold open. + The service role can find open holds by reading `credit_reservations` where + `state = 'held'` (indexed by account, ordered by `created_at`) and then settle or + release them with the recorded key. An API or operator view is a later slice. + +## Verification + +Disposable cluster only (synthetic balances, Unix socket, no hosted credentials): + +```sh +python3 -m unittest discover -s tests -p 'test_credit_reservations.py' +python3 -m unittest discover -s tests -p 'test_credit_reservation_composition.py' +``` + +The original nineteen-test baseline applies only the original reservation migration +and preserves its historical gap demonstration. It covers holds without debit, 12-way concurrent reservations against +one balance, concurrent duplicate reservations and settlements, lost-reply replay, +changed-input conflicts, settlement through the shared charge receipt, over-limit +settlement, release and its state transitions, missing reservations, per-owner +isolation, an atomic top-up concurrent with settlement, an atomic refill with an +outstanding hold, legacy debit concurrency (documenting that it ignores holds), +invalid inputs, missing/duplicate wallets, denied browser roles, denied service-role +deletes and immutable hold columns, and rollback. Locally passed on +PostgreSQL 15 and 17; CI runs 15 and 16. The fixture is the minimal legacy +schema plus the actual charge-receipt migration, not a replay of every migration. + +The composition suite replays the actual charge, reservation, pack and corrective +migrations in order on a synthetic minimal schema. It checks shared/pack holds, +legacy and stable debits, actual refill/top-up RPCs, concurrent admission, settlement +with other holds, lost acknowledgements, rollback, owner isolation, wallet lifecycle, +RLS-hidden holds and the repeatable-read race. It is not a full production schema replay. + +## Release + +Apply through the normal database release workflow before deploying the API +adapters. Verify the table and function grants after release (service role only; +no browser grants). Do not exercise these functions against a real wallet to test +deployment. Rolling back the API stops new callers; keep the table and its rows. diff --git a/tests/test_credit_reservation_composition.py b/tests/test_credit_reservation_composition.py new file mode 100644 index 0000000..723d36d --- /dev/null +++ b/tests/test_credit_reservation_composition.py @@ -0,0 +1,236 @@ +"""Released pack/refill/debit RPCs composed with shared reservations, on local PostgreSQL.""" +import concurrent.futures +import json +import subprocess +import uuid +from credit_charge_fixture import CreditChargeFixture, ROOT + + +class CreditReservationComposition(CreditChargeFixture): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.sql('''CREATE TABLE public.accounts(id uuid PRIMARY KEY); + CREATE TABLE public.account_organization_ids(account_id uuid, organization_id uuid); + ALTER TABLE public.credits_usage ADD COLUMN timestamp timestamp DEFAULT now();''') + # Replay the owning migrations in deployment order, not rewritten model functions. + for name in ['20261010210500_credit_reservations.sql', + '20261020150000_release_pack_jobs.sql', + '20261117000000_compose_credit_reservations.sql']: + cls.sql((ROOT / 'supabase/migrations' / name).read_text()) + + def setUp(self): + self.owner = str(uuid.uuid4()) + self.job = str(uuid.uuid4()) + self.sql(f"INSERT INTO accounts VALUES ('{self.owner}'); INSERT INTO credits_usage" + f"(account_id,remaining_credits) VALUES ('{self.owner}',1000)") + + def rpc(self, function, args, succeeds=True): + result = self.sql(f'SET ROLE service_role; SELECT {function}({args});', succeeds=succeeds) + return json.loads(result.stdout) if succeeds else result + + def reserve(self, amount=600, key='shared'): + return self.rpc('reserve_credits_once', f"'{self.owner}','{key}',{amount}") + + def settle(self, amount=600, key='shared', event=None, succeeds=True): + data = json.dumps(event or {}).replace("'", "''") + return self.rpc('settle_credit_reservation', + f"'{self.owner}','{key}',{amount},'{data}'::jsonb", succeeds) + + def wallet(self): + return int(self.sql(f"SELECT remaining_credits FROM credits_usage WHERE account_id='{self.owner}'").stdout) + + def holds(self): + return int(self.sql(f"SELECT coalesce(sum(credits_held),0) FROM credit_reservations " + f"WHERE account_id='{self.owner}' AND state='held'").stdout) + + def events(self): + return int(self.sql(f"SELECT count(*) FROM usage_events WHERE account_id='{self.owner}'").stdout) + + def command(self, action, data=None, succeeds=True): + payload = json.dumps(data or {}).replace("'", "''") + return self.rpc('release_pack_command', + f"'{self.owner}',null,'{action}','{self.job}','{payload}'::jsonb", succeeds) + + def quote_pack(self): + assets = [{'kind': 'artwork', 'ordinal': 0, 'credits': 100, 'maximum_usd': 0.01}] + self.command('prepare', {'idempotency_key': self.job, 'fingerprint': 'a'*64, + 'brief': {}, 'deliverables': {}, 'status': 'ready_to_quote', 'assets': assets}) + quote = {'id': str(uuid.uuid4()), 'revision': 1, 'credits': 100, + 'maximum_usd': 0.01, 'expires_at': '2099-01-01T00:00:00Z'} + self.command('estimate', {'quote': quote, 'assets': assets}) + return {'quote_id': quote['id'], 'expected_revision': 1, 'approved_max_usd': 0.01} + + def debit_sql(self, amount): + return (f"SET ROLE service_role; SELECT deduct_credits_with_audit('{self.owner}'," + f"{amount}::bigint,'legacy-{uuid.uuid4()}','{{}}'::jsonb);") + + def test_pack_hold_and_shared_hold_protect_the_same_wallet_without_double_counting(self): + self.command('approve', self.quote_pack()) + self.assertEqual(self.reserve()['spendableCredits'], 300) + denied = self.sql(self.debit_sql(301), succeeds=False) + self.assertNotEqual(denied.returncode, 0) + self.assertEqual((self.wallet(), self.holds(), self.events()), (900, 600, 0)) + self.sql(self.debit_sql(300)) + self.command('settle', {'error': 'cancelled before provider dispatch'}) + self.assertEqual(self.wallet(), 700) + self.settle() + self.assertEqual((self.wallet(), self.holds(), self.events()), (100, 0, 2)) + + def test_pack_approval_cannot_consume_a_preexisting_shared_reservation(self): + approval = self.quote_pack() + self.reserve(960) + denied = self.command('approve', approval, succeeds=False) + self.assertNotEqual(denied.returncode, 0) + self.assertEqual(self.command('read')['job']['status'], 'awaiting_approval') + self.assertEqual((self.wallet(), self.holds()), (1000, 960)) + + def test_atomic_adjustment_and_stable_charge_also_cannot_consume_held_funds(self): + self.reserve(960) + for function, args in [ + ('adjust_credit_balance_atomic', f"'{self.owner}',-41"), + ('record_credit_charge_once', f"'{self.owner}','other',41,'{{}}'::jsonb"), + ]: + self.assertNotEqual(self.rpc(function, args, False).returncode, 0) + self.assertEqual((self.wallet(), self.holds(), self.events()), (1000, 960, 0)) + + def test_settlement_excludes_its_own_hold_but_preserves_other_holds(self): + self.reserve(200) + self.reserve(800, 'other') + self.assertEqual(self.settle(150)['creditsReleased'], 50) + self.assertEqual((self.wallet(), self.holds()), (850, 800)) + self.settle(800, 'other') + self.assertEqual((self.wallet(), self.holds(), self.events()), (50, 0, 2)) + + def test_lost_settlement_acknowledgement_reuses_one_charge_and_conflict_rolls_back(self): + self.reserve() + first = self.settle(120) # Committed response is deliberately not used to retry a provider. + replay = self.settle(120) + self.assertEqual(replay['state'], 'reused') + self.assertEqual(replay['eventId'], first['eventId']) + self.assertNotEqual(self.settle(121, succeeds=False).returncode, 0) + self.assertEqual((self.wallet(), self.holds(), self.events()), (880, 0, 1)) + + def test_rejected_charge_restores_open_hold_and_no_event(self): + self.reserve() + self.assertNotEqual(self.settle(120, event={'unexpected': True}, succeeds=False).returncode, 0) + self.assertEqual((self.wallet(), self.holds(), self.events()), (1000, 600, 0)) + self.assertEqual(self.reserve()['status'], 'held') + + def test_actual_refill_counts_pack_hold_once_and_does_not_release_shared_hold(self): + self.sql(f"UPDATE credits_usage SET remaining_credits=150 WHERE account_id='{self.owner}'") + self.command('approve', self.quote_pack()) + self.reserve(40) + stamp = self.sql(f"SELECT timestamp::text FROM credits_usage WHERE account_id='{self.owner}'").stdout.strip() + receipt = self.rpc('refill_credits_to_floor', f"'{self.owner}',200,'{stamp}'::timestamp") + self.assertEqual(receipt['remainingCredits'], 100) + self.assertEqual(self.holds(), 40) + self.settle(30) + stale = self.rpc('refill_credits_to_floor', f"'{self.owner}',200,'{stamp}'::timestamp") + self.assertEqual(stale['state'], 'superseded') + self.command('settle', {'error': 'cancelled'}) + self.assertEqual((self.wallet(), self.holds()), (170, 0)) + + def test_actual_topup_and_settlement_concurrently_both_survive(self): + self.reserve() + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + a = pool.submit(self.rpc, 'increment_credits_atomic', f"'{self.owner}',500") + b = pool.submit(self.settle, 120) + a.result(); b.result() + self.assertEqual((self.wallet(), self.holds(), self.events()), (1380, 0, 1)) + + def test_concurrent_legacy_debit_and_reservation_cannot_both_oversubscribe(self): + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + a = pool.submit(self.reserve, 960) + b = pool.submit(self.sql, self.debit_sql(50), False) + reservation, debit = a.result(), b.result() + if reservation['state'] == 'held': + self.assertNotEqual(debit.returncode, 0) + self.assertEqual((self.wallet(), self.holds(), self.events()), (1000, 960, 0)) + else: + self.assertEqual(reservation['state'], 'insufficient') + self.assertEqual(debit.returncode, 0) + self.assertEqual((self.wallet(), self.holds(), self.events()), (950, 0, 1)) + + def test_existing_no_hold_legacy_debit_behavior_is_unchanged(self): + self.sql(self.debit_sql(1001)) + self.assertEqual((self.wallet(), self.events()), (-1, 1)) + + def test_outstanding_hold_blocks_null_balance_owner_change_and_wallet_deletion(self): + self.reserve() + for statement in [ + f"UPDATE credits_usage SET remaining_credits=NULL WHERE account_id='{self.owner}'", + f"UPDATE credits_usage SET account_id='{uuid.uuid4()}' WHERE account_id='{self.owner}'", + f"DELETE FROM credits_usage WHERE account_id='{self.owner}'", + ]: + self.assertNotEqual(self.sql(statement, succeeds=False).returncode, 0) + self.assertEqual((self.wallet(), self.holds()), (1000, 600)) + + def test_release_keeps_balance_and_makes_capacity_available_to_legacy_debit(self): + self.reserve() + self.rpc('release_credit_reservation', f"'{self.owner}','shared'") + self.sql(self.debit_sql(1000)) + self.assertEqual((self.wallet(), self.holds(), self.events()), (0, 0, 1)) + + def test_old_repeatable_read_snapshot_cannot_miss_a_new_reservation(self): + client = subprocess.Popen(['psql', '-h', self.tmp.name, '-p', '55474', '-d', 'postgres', + '-XAtq', '-v', 'ON_ERROR_STOP=1'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.addCleanup(lambda: client.poll() is None and client.kill()) + client.stdin.write(f"BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT remaining_credits " + f"FROM credits_usage WHERE account_id='{self.owner}';\n") + client.stdin.flush() + self.assertEqual(client.stdout.readline().strip(), '1000') + self.reserve(960) + _, stderr = client.communicate(self.debit_sql(50) + ' COMMIT;', timeout=10) + self.assertNotEqual(client.returncode, 0) + self.assertIn('could not serialize access', stderr) + self.assertEqual((self.wallet(), self.holds(), self.events()), (1000, 960, 0)) + + def test_guard_sees_holds_hidden_from_a_wallet_writer(self): + role = 'wallet_' + uuid.uuid4().hex + self.sql(f"CREATE ROLE {role}; GRANT USAGE ON SCHEMA public TO {role}; " + f"GRANT SELECT, UPDATE, DELETE ON credits_usage TO {role}; " + f"CREATE POLICY {role}_policy ON credits_usage TO {role} USING (true) WITH CHECK (true);") + self.reserve(960) + self.assertNotEqual(self.sql(f'SET ROLE {role}; SELECT * FROM credit_reservations', False).returncode, 0) + # Unrelated metadata updates are not a financial operation. + self.sql(f"SET ROLE {role}; UPDATE credits_usage SET timestamp=now() WHERE account_id='{self.owner}'") + denied = self.sql(f"SET ROLE {role}; UPDATE credits_usage SET remaining_credits=950 " + f"WHERE account_id='{self.owner}'", False) + self.assertNotEqual(denied.returncode, 0) + self.assertIn('Outstanding credit reservations', denied.stderr) + self.assertNotEqual(self.sql(f'SET ROLE {role}; SELECT protect_credit_reservations()', False).returncode, 0) + self.assertEqual((self.wallet(), self.holds()), (1000, 960)) + + def test_concurrent_pack_approval_and_shared_reservation_cannot_both_oversubscribe(self): + approval = self.quote_pack() + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + a = pool.submit(self.reserve, 960) + b = pool.submit(self.command, 'approve', approval, False) + reservation, pack = a.result(), b.result() + if reservation['state'] == 'held': + self.assertNotEqual(pack.returncode, 0) + self.assertEqual((self.wallet(), self.holds()), (1000, 960)) + else: + self.assertEqual(pack.returncode, 0) + self.assertEqual((self.wallet(), self.holds()), (900, 0)) + + def test_failed_usage_insert_restores_hold_and_wallet(self): + name = 'fail_usage_' + uuid.uuid4().hex + self.sql(f"""CREATE FUNCTION {name}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.provider='fixture_failure' THEN RAISE EXCEPTION 'fixture insert failure'; + END IF; RETURN NEW; END $$; + CREATE TRIGGER {name} BEFORE INSERT ON usage_events FOR EACH ROW EXECUTE FUNCTION {name}();""") + self.reserve() + rejected = self.settle(120, event={'provider': 'fixture_failure'}, succeeds=False) + self.assertIn('fixture insert failure', rejected.stderr) + self.assertEqual((self.wallet(), self.holds(), self.events()), (1000, 600, 0)) + self.assertEqual(self.reserve()['status'], 'held') + + def test_another_owners_holds_do_not_limit_this_wallet(self): + other = str(uuid.uuid4()) + self.sql(f"INSERT INTO credits_usage(account_id,remaining_credits) VALUES ('{other}',1000)") + self.rpc('reserve_credits_once', f"'{other}','other',1000") + self.sql(self.debit_sql(1000)) + self.assertEqual((self.wallet(), self.holds()), (0, 0)) diff --git a/tests/test_credit_reservations.py b/tests/test_credit_reservations.py new file mode 100644 index 0000000..674154d --- /dev/null +++ b/tests/test_credit_reservations.py @@ -0,0 +1,264 @@ +"""Credit reservations (holds) on the existing wallet, against real local transactions.""" +import concurrent.futures +import json +import uuid +from credit_charge_fixture import CreditChargeFixture, ROOT + +MIGRATION = ROOT / 'supabase/migrations/20261010210500_credit_reservations.sql' + + +class CreditReservations(CreditChargeFixture): + @classmethod + def setUpClass(cls): + super().setUpClass() + # Required, never skipped: without the migration the suite must fail. + cls.sql(MIGRATION.read_text()) + + def setUp(self): + self.owner = str(uuid.uuid4()) + self.sql(f"INSERT INTO credits_usage(account_id,remaining_credits) VALUES ('{self.owner}',1000)") + + @staticmethod + def quote(value): + return value.replace("'", "''") + + def reserve_statement(self, key='work-1', credits=300, owner=None, role='service_role'): + return (f"SET ROLE {role}; SELECT public.reserve_credits_once(" + f"'{owner or self.owner}', '{self.quote(key)}', {credits});") + + def settle_statement(self, key='work-1', credits=120, event=None, owner=None): + payload = self.quote(json.dumps(event or {})) + return (f"SET ROLE service_role; SELECT public.settle_credit_reservation(" + f"'{owner or self.owner}', '{self.quote(key)}', {credits}, '{payload}'::jsonb);") + + def release_statement(self, key='work-1', owner=None): + return (f"SET ROLE service_role; SELECT public.release_credit_reservation(" + f"'{owner or self.owner}', '{self.quote(key)}');") + + def run_json(self, statement): + return json.loads(self.sql(statement).stdout) + + def reserve(self, **kwargs): + return self.run_json(self.reserve_statement(**kwargs)) + + def settle(self, **kwargs): + return self.run_json(self.settle_statement(**kwargs)) + + def release(self, **kwargs): + return self.run_json(self.release_statement(**kwargs)) + + def failing(self, statement): + result = self.sql(statement, succeeds=False) + self.assertNotEqual(result.returncode, 0) + return result.stderr + + def balance(self, owner=None): + return int(self.sql(f"SELECT remaining_credits FROM credits_usage " + f"WHERE account_id='{owner or self.owner}'").stdout) + + def holds(self, state='held'): + return int(self.sql(f"SELECT coalesce(sum(credits_held),0) FROM credit_reservations " + f"WHERE account_id='{self.owner}' AND state='{state}'").stdout) + + def usage_events(self): + return int(self.sql(f"SELECT count(*) FROM usage_events WHERE account_id='{self.owner}'").stdout) + + def test_hold_reduces_spendable_without_debit_or_usage_event(self): + receipt = self.reserve() + self.assertEqual(receipt['state'], 'held') + self.assertRegex(receipt['reservationId'], r'^hold-v1-[a-f0-9]{64}$') + self.assertEqual(receipt['creditsHeld'], 300) + self.assertEqual(receipt['spendableCredits'], 700) + # A hold is not a temporary debit: refill/auto-top-up never see it as consumption. + self.assertEqual(self.balance(), 1000) + self.assertEqual(self.usage_events(), 0) + denied = self.reserve(key='work-2', credits=701) + self.assertEqual(denied, {'state': 'insufficient', 'creditsRequested': 701, 'spendableCredits': 700}) + self.assertEqual(self.holds(), 300) + + def test_concurrent_reservations_never_exceed_the_balance(self): + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + receipts = list(pool.map(lambda i: self.reserve(key=f'work-{i}', credits=100), range(12))) + self.assertEqual(sum(r['state'] == 'held' for r in receipts), 10) + self.assertEqual(sum(r['state'] == 'insufficient' for r in receipts), 2) + self.assertEqual(self.holds(), 1000) + self.assertEqual(self.balance(), 1000) + + def test_concurrent_duplicate_reservation_holds_once(self): + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + receipts = list(pool.map(lambda _: self.reserve(), range(12))) + self.assertEqual(sum(r['state'] == 'held' for r in receipts), 1) + self.assertEqual(sum(r['state'] == 'reused' for r in receipts), 11) + self.assertEqual(len({r['reservationId'] for r in receipts}), 1) + self.assertEqual(self.holds(), 300) + + def test_replayed_reservation_reports_current_status_and_changed_amount_conflicts(self): + first = self.reserve() + self.assertEqual(self.reserve(), {'state': 'reused', 'reservationId': first['reservationId'], + 'status': 'held', 'creditsHeld': 300}) + self.assertIn('Credit reservation identity conflict', self.failing(self.reserve_statement(credits=301))) + self.release() + # A replay after release never re-holds capacity for the same work. + self.assertEqual(self.reserve()['status'], 'released') + self.assertEqual(self.holds(), 0) + + def test_settlement_charges_actual_credits_and_releases_the_remainder(self): + held = self.reserve() + receipt = self.settle(event={'model_id': 'fixture', 'output_tokens': 12}) + self.assertEqual(receipt['state'], 'settled') + self.assertEqual(receipt['reservationId'], held['reservationId']) + self.assertRegex(receipt['eventId'], r'^charge-v1-[a-f0-9]{64}$') + self.assertEqual((receipt['creditsHeld'], receipt['creditsCharged'], receipt['creditsReleased']), + (300, 120, 180)) + self.assertEqual(self.balance(), 880) + self.assertEqual(self.holds(), 0) + self.assertEqual(self.holds('settled'), 300) + row = self.sql(f"SELECT credits_deducted, model_id, output_tokens FROM usage_events " + f"WHERE id='{receipt['eventId']}'").stdout.strip() + self.assertEqual(row, '120|fixture|12') + self.assertEqual(self.reserve(key='work-2', credits=880)['state'], 'held') + + def test_settlement_shares_the_charge_receipt_identity(self): + self.reserve() + settled = self.settle() + direct = self.run_json(f"SET ROLE service_role; SELECT public.record_credit_charge_once(" + f"'{self.owner}', 'work-1', 120, '{{}}'::jsonb);") + self.assertEqual(direct['state'], 'reused') + self.assertEqual(direct['eventId'], settled['eventId']) + self.assertEqual(self.balance(), 880) + + def test_lost_and_concurrent_settlement_acknowledgements_charge_once(self): + self.reserve() + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + receipts = list(pool.map(lambda _: self.settle(), range(12))) + self.assertEqual(sum(r['state'] == 'settled' for r in receipts), 1) + self.assertEqual(sum(r['state'] == 'reused' for r in receipts), 11) + self.assertEqual(len({r['eventId'] for r in receipts}), 1) + self.assertEqual(self.settle()['state'], 'reused') + self.assertEqual(self.balance(), 880) + self.assertEqual(self.usage_events(), 1) + + def test_changed_settlement_conflicts_and_keeps_the_first_charge(self): + self.reserve() + self.settle() + for args in [{'credits': 121}, {'event': {'provider': 'different'}}]: + self.assertIn('Credit charge identity conflict', self.failing(self.settle_statement(**args))) + self.assertEqual(self.balance(), 880) + + def test_settlement_above_the_reservation_is_rejected_and_the_hold_remains(self): + self.reserve() + self.assertIn('Credit settlement exceeds reservation', self.failing(self.settle_statement(credits=301))) + self.assertEqual(self.balance(), 1000) + self.assertEqual(self.holds(), 300) + self.assertEqual(self.usage_events(), 0) + + def test_release_returns_capacity_without_charge_and_closes_the_reservation(self): + self.reserve() + receipt = self.release() + self.assertEqual((receipt['state'], receipt['creditsHeld'], receipt['creditsReleased']), + ('released', 300, 300)) + self.assertEqual(self.release()['state'], 'reused') + self.assertEqual((self.balance(), self.holds(), self.usage_events()), (1000, 0, 0)) + self.assertIn('Credit reservation was released', self.failing(self.settle_statement())) + self.reserve(key='work-2') + self.settle(key='work-2') + self.assertIn('Credit reservation was settled', self.failing(self.release_statement(key='work-2'))) + self.assertEqual(self.balance(), 880) + + def test_settle_or_release_without_a_reservation_never_charges(self): + self.assertIn('Credit reservation not found', self.failing(self.settle_statement())) + self.assertIn('Credit reservation not found', self.failing(self.release_statement())) + self.assertEqual((self.balance(), self.usage_events()), (1000, 0)) + + def test_holds_are_isolated_per_billing_owner(self): + other = str(uuid.uuid4()) + self.sql(f"INSERT INTO credits_usage(account_id,remaining_credits) VALUES ('{other}',100)") + mine = self.reserve(credits=1000) + # Same operation key, different billing owner: a separate hold and identity. + foreign = self.reserve(owner=other, credits=100) + self.assertEqual((mine['state'], foreign['state']), ('held', 'held')) + self.assertNotEqual(foreign['reservationId'], mine['reservationId']) + self.assertEqual(self.reserve(owner=other, key='work-2', credits=1)['state'], 'insufficient') + self.assertIn('Credit reservation not found', self.failing(self.release_statement(owner=other, key='absent'))) + + def test_atomic_top_up_and_settlement_concurrently_both_survive(self): + # Models an in-place (atomic) top-up writer such as database#101's. Main's API + # top-up and refill still write an absolute balance read earlier, which can erase + # a settlement that commits in between; callers stay disconnected until that lands. + self.reserve() + top_up = (f"SET ROLE service_role; UPDATE credits_usage SET remaining_credits = remaining_credits + 500 " + f"WHERE account_id='{self.owner}'") + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(self.sql, [top_up, self.settle_statement()])) + self.assertEqual(self.balance(), 1380) + + def test_atomic_refill_with_an_outstanding_hold_keeps_the_hold(self): + # Models an in-place refill to the plan floor (database#101), not main's API refill. + self.sql(f"UPDATE credits_usage SET remaining_credits = 200 WHERE account_id='{self.owner}'") + self.reserve(credits=150) + self.sql(f"UPDATE credits_usage SET remaining_credits = GREATEST(remaining_credits, 1000) " + f"WHERE account_id='{self.owner}'") + self.assertEqual(self.holds(), 150) + self.assertEqual(self.reserve(key='work-2', credits=851)['spendableCredits'], 850) + + def test_legacy_debit_concurrent_with_reservation_is_kept_but_ignores_holds(self): + legacy = (f"SET ROLE service_role; SELECT deduct_credits_with_audit(" + f"'{self.owner}',50::bigint,'legacy-{self.owner}','{{}}'::jsonb)") + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(self.sql, [legacy, self.reserve_statement(credits=960)])) + self.assertEqual(self.balance(), 950) + state = json.loads(results[1].stdout)['state'] + self.assertIn(state, {'held', 'insufficient'}) + # Known limitation: legacy debit callers do not consult holds, so a debit that + # commits after the hold can leave spendable credit below zero until settlement. + self.assertEqual(self.holds(), 960 if state == 'held' else 0) + + def test_invalid_inputs_and_missing_or_duplicate_wallets_never_write(self): + statements = [self.reserve_statement(credits=0), self.reserve_statement(credits=-1), + self.reserve_statement(key=' '), self.reserve_statement(key='k' * 201), + self.reserve_statement(credits=9007199254740992), + self.reserve_statement(owner=str(uuid.uuid4()))] + for statement in statements: + self.failing(statement) + self.reserve() + for statement in [self.settle_statement(credits=0), self.settle_statement(event={'extra': True}), + self.settle_statement(event={'input_tokens': -1}), self.release_statement(key='')]: + self.failing(statement) + self.assertEqual((self.balance(), self.holds(), self.usage_events()), (1000, 300, 0)) + self.sql(f"INSERT INTO credits_usage(account_id,remaining_credits) VALUES ('{self.owner}',700)") + for statement in [self.reserve_statement(key='work-2'), self.settle_statement(), self.release_statement()]: + self.assertIn('Credit wallet is unavailable or ambiguous', self.failing(statement)) + self.assertEqual(self.holds(), 300) + + def test_browser_roles_are_denied_and_holds_cannot_be_deleted(self): + self.reserve() + for role in ['anon', 'authenticated']: + for statement in [self.reserve_statement(key='browser'), self.settle_statement(), + self.release_statement(), 'SELECT * FROM credit_reservations;']: + result = self.sql(statement.replace('service_role', role) if 'SET ROLE' in statement + else f'SET ROLE {role}; {statement}', succeeds=False) + self.assertIn('permission denied', result.stderr) + self.assertEqual((self.balance(), self.holds()), (1000, 300)) + for statement in ['DELETE FROM credit_reservations;', 'TRUNCATE credit_reservations;']: + self.assertIn('permission denied', self.failing(f'SET ROLE service_role; {statement}')) + self.assertEqual(self.holds(), 300) + + def test_service_role_cannot_rewrite_a_hold_owner_key_or_amount(self): + self.reserve() + for column, value in [('credits_held', '1'), ('account_id', f"'{uuid.uuid4()}'"), + ('operation_key', "'other'"), ('id', f"'hold-v1-{'0' * 64}'"), + ('created_at', 'now()')]: + self.assertIn('permission denied', self.failing( + f'SET ROLE service_role; UPDATE credit_reservations SET {column} = {value};')) + open_holds = self.sql(f"SET ROLE service_role; SELECT operation_key, credits_held FROM " + f"credit_reservations WHERE account_id='{self.owner}' AND state='held' " + f"ORDER BY created_at;").stdout.strip() + self.assertEqual(open_holds, 'work-1|300') + self.assertEqual(self.settle()['state'], 'settled') + + def test_transaction_rollback_leaves_no_hold_or_charge(self): + self.sql('BEGIN;' + self.reserve_statement() + 'ROLLBACK;') + self.assertEqual(self.holds(), 0) + self.reserve() + self.sql('BEGIN;' + self.settle_statement() + 'ROLLBACK;') + self.assertEqual((self.balance(), self.holds(), self.usage_events()), (1000, 300, 0))