diff --git a/.github/workflows/stream-binding-proposal.yml b/.github/workflows/stream-binding-proposal.yml index c93429b..98b458b 100644 --- a/.github/workflows/stream-binding-proposal.yml +++ b/.github/workflows/stream-binding-proposal.yml @@ -4,6 +4,7 @@ on: branches: [main] paths: - "proposals/stream-evidence-binding-2207/**" + - "proposals/stream-retention-writer-2207/**" - "tests/test_release_cases.py" - "supabase/migrations/**" - ".github/workflows/stream-binding-proposal.yml" @@ -34,4 +35,4 @@ jobs: timeout-minutes: 3 run: | export PATH="/usr/lib/postgresql/${{ matrix.postgres }}/bin:$PATH" - python3 proposals/stream-evidence-binding-2207/test_binding.py + python3 proposals/stream-retention-writer-2207/test_writer.py diff --git a/proposals/stream-evidence-binding-2207/PROTOCOL.md b/proposals/stream-evidence-binding-2207/PROTOCOL.md index 81ab557..55f916a 100644 --- a/proposals/stream-evidence-binding-2207/PROTOCOL.md +++ b/proposals/stream-evidence-binding-2207/PROTOCOL.md @@ -14,6 +14,15 @@ Its fixture inserts are not a retention writer. It does not authorize derived su growth or comparison fields: exact bundle equality is not a result-field validator. Root and Catalog review the revised exact heads before any further implementation. +## Follow-on offline writer + +The [executable retention writer proposal](../stream-retention-writer-2207/PROTOCOL.md) +now exercises the transaction described below outside migrations. It shares the +extracted authority helper and adds logical row/history budgets to this reader. +The original 25 read regressions remain; production writer activation and derived +field authorization are still excluded. The original fixture remains a read fixture, +not evidence that the separate writer ran. + ## Existing source authority Database base `6ae03a0cc71dc2ad3955594c9cfcb6156c3fb8b8`; API inspection base @@ -108,7 +117,7 @@ Those need the strict writer/result-field validator using existing `compareSpotifyDailyStreams` semantics before any field ObjectRef is enabled. Incomplete evidence may be retained without authorizing invented numeric totals. -## Proposed retention transaction (not implemented) +## Retention transaction design (offline follow-on linked above) 1. Derive actor/owner from trusted authentication; strictly validate exact request, subject, catalog, periods, expected revision and observed tuple set. The descriptor @@ -135,8 +144,9 @@ Incomplete evidence may be retained without authorizing invented numeric totals. 6. Return exact result/topic/revision/version receipt. Unknown acknowledgment requires receipt reconciliation, not provider retry or implicit collection. -Concurrent first creation, CAS/retry/rollback and receipt behavior are not implemented -or proven by direct fixture inserts. They remain required before runtime work. +Concurrent first creation, CAS/retry/rollback and receipt behavior are not proven +by this read fixture. The separate executable writer suite now qualifies those +operations offline; runtime integration remains gated. ## Read/lock qualification and withdrawal @@ -174,6 +184,6 @@ fail-closed overflow behavior. No production grants/RLS delivery, all cascade interleavings, production writer idempotency/CAS/rollback, actual API transport byte validation, derived field validation or durable chat composition is qualified. -Next: Catalog independently reviews alignment to adopted #1187; root reviews revised -exact heads and fresh CI. Only after exact protocol adoption may the executable -retention writer be built and qualified. Source merge alone activates nothing. +The evidence protocol was adopted in Database #127. Current next work and writer +qualification are recorded in the linked follow-on protocol; root retains review +and activation authority. Source merge alone activates nothing. diff --git a/proposals/stream-evidence-binding-2207/proposal.sql b/proposals/stream-evidence-binding-2207/proposal.sql index 0fa4a1f..28b8083 100644 --- a/proposals/stream-evidence-binding-2207/proposal.sql +++ b/proposals/stream-evidence-binding-2207/proposal.sql @@ -2,38 +2,59 @@ -- No tables, runtime call sites, service grants, collection or implicit expansion. create schema stream_binding_proposal; revoke all on schema stream_binding_proposal from public,anon,authenticated,service_role; -create function stream_binding_proposal.read_binding( - p_actor uuid,p_owner uuid,p_request uuid,p_subject uuid,p_catalog uuid, - p_result uuid,p_version uuid,p_revision bigint -) returns jsonb language plpgsql set search_path='' set timezone='UTC' as $$ -declare - isrc_value text; resolved jsonb; payload jsonb; actual jsonb; source_key uuid; - first_day date; last_day date; topic_value text; source_uri text; - midpoint date; days_count integer; provider_id text; expected jsonb; envelope jsonb; - absent_days jsonb; null_days jsonb; earliest timestamptz; latest timestamptz; version_time timestamptz; +create function stream_binding_proposal.within_budget(p_owner uuid,p_catalog uuid,p_subject uuid,p_isrc text,p_since date,p_until date) +returns boolean language plpgsql set search_path='' as $$ +declare n bigint; bytes bigint; uri text; +begin + if p_since is null or p_until is null or not isfinite(p_since) or not isfinite(p_until) + or p_until-p_since not between 2 and 732 or (p_until-p_since)%2<>0 or p_until>current_date + then return false; end if; + -- Cap visited matching history rows before locking/aggregating full evidence. + select count(*),coalesce(sum(pg_column_size(x)+octet_length(x.provider_recording_id)+octet_length(x.source_hash)),0) + into n,bytes from (select o.* from public.catalog_stream_observations o + where o.catalog_id=p_catalog and o.isrc=p_isrc and o.date>=p_since and o.date4096 or bytes>8388608 then return false; end if; + uri:='urn:recoup:retained-catalog-streams:'||p_catalog::text||':'||p_subject::text||':all_dsps:worldwide'; + select count(*) into n from (select 1 from public.context_sources + where owner_id=p_owner and kind='provider_metadata' and source_url=uri limit 2) x; + if n>1 then return false; end if; + select count(*),coalesce(sum(octet_length(x.content::text)),0) into n,bytes from ( + select v.content from public.context_source_versions v join public.context_sources s on s.id=v.source_id and s.owner_id=v.owner_id + where s.owner_id=p_owner and s.source_url=uri and s.kind='provider_metadata' limit 257) x; + if n>256 or bytes>8388608 then return false; end if; + select count(*) into n from (select 1 from public.context_result_sources rs + join public.context_source_versions v on v.id=rs.source_version_id and v.owner_id=rs.owner_id + join public.context_sources s on s.id=v.source_id and s.owner_id=v.owner_id + where s.owner_id=p_owner and s.source_url=uri and s.kind='provider_metadata' limit 513) x; + return n<=512; +end $$; +revoke all on function stream_binding_proposal.within_budget(uuid,uuid,uuid,text,date,date) from public,anon,authenticated,service_role; +create function stream_binding_proposal.lock_scope(p_actor uuid,p_owner uuid,p_request uuid,p_subject uuid,p_catalog uuid) +returns text language plpgsql set search_path='' as $$ +declare isrc_value text; resolved jsonb; begin - if current_setting('transaction_isolation') <> 'read committed' then return 'null'::jsonb; end if; + if current_setting('transaction_isolation') <> 'read committed' then return null; end if; -- Parent order follows commit_catalog_stream_track. Chat/session locks, if any, -- must be acquired by a future durable caller before entering this boundary. perform 1 from public.accounts where id=p_owner for key share; - if not found then return 'null'::jsonb; end if; + if not found then return null; end if; perform 1 from public.catalogs where id=p_catalog for key share; - if not found then return 'null'::jsonb; end if; + if not found then return null; end if; select song_isrc into isrc_value from public.context_subjects where id=p_subject and kind='recording'; - if isrc_value is null then return 'null'::jsonb; end if; + if isrc_value is null then return null; end if; perform 1 from public.songs where isrc=isrc_value for key share; perform 1 from public.catalog_stream_tracking where catalog_id=p_catalog and owner_id=p_owner for update; - if not found then return 'null'::jsonb; end if; + if not found then return null; end if; perform public.authorize_context_case_actor(p_actor,p_owner); perform 1 from public.account_catalogs where account=p_owner and catalog=p_catalog for share; - if not found then return 'null'::jsonb; end if; + if not found then return null; end if; perform 1 from public.catalog_songs where catalog=p_catalog and song=isrc_value for share; - if not found then return 'null'::jsonb; end if; + if not found then return null; end if; perform 1 from public.context_requests where id=p_request and owner_id=p_owner for share; perform 1 from public.context_request_catalog_members where owner_id=p_owner and request_id=p_request and recording_subject_id=p_subject and catalog_id=p_catalog and song_isrc=isrc_value for share; - if not found then return 'null'::jsonb; end if; + if not found then return null; end if; -- Lock all mutable identity participants before invoking the existing exact resolver. perform 1 from public.context_resources r join public.context_requests q on q.resource_id=r.id where q.id=p_request for share of r; @@ -42,7 +63,22 @@ begin where m.owner_id=p_owner and m.request_id=p_request and m.recording_subject_id=p_subject order by s.id for share of s; resolved:=public.resolve_context_catalog_member_recording(p_owner,p_request,p_subject); - if resolved->>'isrc' is distinct from isrc_value then return 'null'::jsonb; end if; + if resolved->>'isrc' is distinct from isrc_value then return null; end if; + return isrc_value; +end $$; +revoke all on function stream_binding_proposal.lock_scope(uuid,uuid,uuid,uuid,uuid) from public,anon,authenticated,service_role; +create function stream_binding_proposal.read_binding( + p_actor uuid,p_owner uuid,p_request uuid,p_subject uuid,p_catalog uuid, + p_result uuid,p_version uuid,p_revision bigint +) returns jsonb language plpgsql set search_path='' set timezone='UTC' as $$ +declare + isrc_value text; resolved jsonb; payload jsonb; actual jsonb; source_key uuid; + first_day date; last_day date; topic_value text; source_uri text; + midpoint date; days_count integer; provider_id text; expected jsonb; envelope jsonb; + absent_days jsonb; null_days jsonb; earliest timestamptz; latest timestamptz; version_time timestamptz; +begin + isrc_value:=stream_binding_proposal.lock_scope(p_actor,p_owner,p_request,p_subject,p_catalog); + if isrc_value is null then return 'null'::jsonb; end if; topic_value:='catalog_retained_streams'; source_uri:='urn:recoup:retained-catalog-streams:'||p_catalog::text||':'||p_subject::text||':all_dsps:worldwide'; -- A replacement active row cannot restore this withdrawn source scope. @@ -83,6 +119,7 @@ begin first_day:=(payload->>'since')::date; last_day:=(payload->>'until')::date; if first_day is null or last_day is null or last_day-first_day not between 2 and 732 or (last_day-first_day)%2<>0 or last_day>current_date then return 'null'::jsonb; end if; + if not stream_binding_proposal.within_budget(p_owner,p_catalog,p_subject,isrc_value,first_day,last_day) then return 'null'::jsonb; end if; days_count:=(last_day-first_day)/2; midpoint:=first_day+days_count; -- Lock retained rows and their run parents against mutation/removal. This does -- not prevent arbitrary direct inserts that bypass the tracking writer protocol. diff --git a/proposals/stream-retention-writer-2207/PROTOCOL.md b/proposals/stream-retention-writer-2207/PROTOCOL.md new file mode 100644 index 0000000..a84a32d --- /dev/null +++ b/proposals/stream-retention-writer-2207/PROTOCOL.md @@ -0,0 +1,158 @@ +# Executable retained-evidence writer proposal (#2207) + +Status: offline source qualification, 2026-10-27. Based on adopted Database #127 +merge `7160086321f16db8a1d4f09c721d3928ade042f6` and API #1188 merge +`d9683853c8471423c5aa763bf60270bee5b89ff7`. The adopted Catalog source contract and +[exact evidence read protocol](../stream-evidence-binding-2207/PROTOCOL.md) remain +in force. This adds an executable writer, not a fixture-only writer claim. + +Everything stays outside migrations in the private `stream_binding_proposal` schema. +Schema usage and all functions remain revoked from public, anon, authenticated and +service_role. No runtime call site, grant, provider request, collection, customer +operation or deployment is introduced. Root owns merge and activation approval. + +## Writer and receipt storage + +Load the read proposal first, then `writer.sql`. `retain(actor,owner,request,subject, +catalog,key,expected_revision,evidence)` takes trusted authenticated scope and the +exact expected evidence bundle, never a provider fetch request. It validates the +bundle through the adopted reader before success. All inserts and the document +revision change roll back if final validation or receipt persistence fails. + +No receipt store or observation mirror is added. Existing records hold: + +- `context_attempts.module = retained-binding:`, attempt 1: retry identity + scoped to the exact owned request. Keys are 1–100 ASCII letters/digits/underscore/ + hyphen. Current Catalog request scope prevents cross-catalog key ambiguity. +- `context_results.reuse_key`: SHA256 of UTC PostgreSQL JSONB text for + `[owner,request,subject,catalog,key,expected_revision,evidence]`. A changed command + under the same key conflicts, including a changed expected revision. +- `context_results.validation.retentionReceipt`: immutable writer receipt with + owner/request/subject/catalog, key, commandDigest, resultId, sourceVersionId and + committed document revision. The writer sets it before returning success. +- `context_source_versions.fingerprint`: separately computed content-only hash. + Different command keys may reuse the same exact version while creating their + own attempt/result/receipt and advancing the document revision. Deduplication is + at the immutable evidence-version level, not by overwriting an earlier receipt. + +Provider `luminate` and recipe `retained-luminate-daily-v1` identify local retention +of existing observations. Attempt times are local binding times. Provider cost +remains unknown, with no confirmed zero or invented provider request ID. Version +retrieved_at remains the latest included retained observation time. + +`reconcile_retention(actor,owner,request,key)` performs no writes and returns the +original receipt only after exact current authority/readback. Lost acknowledgments +can therefore be reconciled without repeating collection or creating another result. +An exact retry takes the same path. Superseded, corrected, withdrawn, removed, +revoked or over-budget bindings return redacted JSON null; the original commit may +still exist historically, so null does not prove it never committed. Do not retry +with a new key automatically. No stale receipt substitutes a newer version. + +Unavailable/invalid evidence returns JSON null and rolls back tentative changes. +Changed command keys and revision mismatches raise explicit command/revision conflict +errors. Operational SQL errors propagate and roll back the statement; they are not +converted to a successful empty receipt. Actor is excluded from the command digest +so another currently authorized workspace actor can reconcile the same owned request. + +## Transaction and lock behavior + +The adopted authority sequence is extracted to `lock_scope`, shared by read and +write: READ COMMITTED only; owner account → catalog → song KEY SHARE; tracking +UPDATE; actor/workspace, workspace/catalog, catalog/song and request/expanded-edge/ +resource/subject checks and SHARE locks. It reuses the exact expanded-member resolver. +There is no implicit expansion or identity creation. + +Tracking serializes competing retention commands for the catalog and the supported +`commit_catalog_stream_track`. Source lookup includes withdrawn history; duplicate or +withdrawn scope refuses. Source UPDATE precedes version and document locks. The +stable `catalog_retained_streams` document unique key serializes first creation and +replacement across different catalogs, even though they have different tracking and +source locks. Expected revision is checked while holding the document UPDATE lock. + +The writer refuses a current customer_assertion before insertion/acceptance. It does +not lock an old result after taking the document lock: that would invert withdrawal's +source → result → document order. Participating writers treat existing evidence and +receipt contents as immutable. Generic/direct mutations are outside this proof. + +The writer inserts/reuses the version, writes the local attempt/result/link, calls +existing `accept_context_result`, then calls the adopted `read_binding`. That final +check verifies exact retained tuples, run/date/hash/time/provider history, closed +schema, byte bounds, current result/revision and full version set. Receipt persistence +follows that check, in the same transaction. A function exception subtransaction +rolls back the entire tentative save on refusal. Synthetic trigger injection verifies +that even a failure at the final receipt update leaves no partial save. + +Future durable composition must acquire session/chat/fence locks before this boundary. +No source function calls back into chat locking. This proposal does not implement +that composition or a durable API endpoint. + +## Logical work budgets and overflow + +`within_budget` is shared by writer preflight and the adopted reader. After holding +tracking, it limits matching retained history in the requested 2–732-date window to +4,096 rows and 8 MiB of measured row/string data. For the exact owned source locator, +it permits at most 256 versions / 8 MiB of JSONB-text content and 512 result-source +links. It inspects at most limit+1 matching rows for each count. The existing +catalog/ISRC/date index and source/version indexes support these predicates. + +Refuse overflow, never delete old history, truncate tuples, bypass versions or hide +lineage to fit. Post-insertion readback repeats these checks, so a writer that would +create version 257 or link 513 rolls back. Exact retries and reconcile also refuse if +current history exceeds the adopted read budget. The existing 512 KiB evidence and +768 KiB response bounds remain separate. Durable command/data-part limits are not +increased. + +These are logical matched-row/byte refusal budgets, **not hard physical scan, CPU, +wall-clock or memory ceilings**. PostgreSQL can scan unrelated rows, and evaluating +an existing oversized JSON value can materialize it before refusal. No statement +or lock timeout is installed by this proposal. Runtime adoption still needs an +execution deadline/work policy and query-plan qualification. No claim of universally +bounded database execution follows from the row caps. + +## Participating writers and enforcement boundary + +The following inventory is based on the adopted Database source, not hosted grants: + +| Path | Participation / enforcement | +| --------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Proposed retain/read/reconcile | Share exact scope helper and tracking lock; immutable evidence plus existing CAS; default-denied application execution. | +| `commit_catalog_stream_track`, current `20261010040000` body | Locks the same parent/tracking/member rows before inserting observations. Real concurrent test verifies contention, and actual correction commit invalidates old receipt. | +| `withdraw_context_source`, foundation | Source UPDATE then results/documents. Held replacement blocks withdrawal; subsequent withdrawal denies retries and new bindings without resurrection. | +| Current membership delete/update | Conflicts with held SHARE rows; later calls reauthorize and refuse removed access. | +| Generic `save_context_metadata` (`20260920040000` replacement) | Takes arbitrary topic/URL, uses active-source lookup and Spotify-specific provenance. It is not admitted to this reserved source/topic protocol. No deployment claim that it is blocked today. | +| `complete_context_enrichment` (`20260923070000`) / current claim allowlist (`20260924020000`) | Existing provider workflow, with different topic/output checks and write ordering; not a retained-stream writer. | +| Release/artist/catalog/supporting-text/campaign entry and identity writers | Write other source/topic families. Must remain excluded from this reserved topic/locator; no implicit delegation to them. | +| Direct service SQL, generic source/result updates and administrative deletion | Existing foundation and stream migrations grant broad service table access. These can bypass the tracking protocol and immutable-content assumption. Explicitly outside atomicity proof. | + +Enforcement **inside this offline proposal** is the default-denied schema/functions, +mandatory shared helper, lock/CAS/readback, and rollback tests. This does not enforce +the allowlist against privileged direct writers. Before any shared runtime claim, +root must inventory current deployed writers and adopt a permission or guarded-write +mechanism that reserves this source/topic and observation write path. No GUC flag, +caller boolean, advisory-lock convention or unreviewed production grant is offered +as proof that privileged bypass is prevented. Current atomicity claims apply only +to the participating operations above. + +## Evidence and remaining work + +Run `python3 proposals/stream-retention-writer-2207/test_writer.py` with one PostgreSQL +installation on PATH. The runner applies actual existing migrations and both proposal +files in a fresh Unix-socket-only cluster. It runs all 25 adopted read regressions +plus 20 writer tests (45 total). Synthetic baseline parents and expected bundles +remain explicit; all writer outcomes come from executing `retain`/reconcile, not +manual insertion of the result under test. + +Writer cases cover first creation; identical retry/reconcile; changed command and +revision conflict; same-key and different-key races; cross-catalog first creation and +replacement CAS; version dedup with separate receipts; prior-document preservation; +customer correction/removed-version refusal; withdrawal/revocation and provider +history; actual stream commit correction; late receipt fault rollback; real held +writer contention with supported mutators; row/byte/history/link budgets and denial +of application execution. No provider data or hosted credentials are used. + +Still excluded: derived totals/growth and field authorization, actual API serialized +byte checks, durable chat composition, production permission enforcement, all parent +cascade interleavings, deployment and live customer outcomes. Catalog supports future +field semantics through the canonical comparison helper; do not duplicate that +comparison calculation in SQL. Root reviews exact source, independent review and +PG15/17 CI before deciding the next scope. Source merge activates nothing. diff --git a/proposals/stream-retention-writer-2207/test_writer.py b/proposals/stream-retention-writer-2207/test_writer.py new file mode 100644 index 0000000..116a6a8 --- /dev/null +++ b/proposals/stream-retention-writer-2207/test_writer.py @@ -0,0 +1,285 @@ +"""Actual offline writer and adopted read regressions; no hosted credentials.""" +import json +import pathlib +import sys +import subprocess +import unittest +from concurrent.futures import ThreadPoolExecutor + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent / 'stream-evidence-binding-2207')) +from test_binding import Binding, uid + + +class Writer(Binding): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.sql((HERE / 'writer.sql').read_text()) + + def setUp(self): + super().setUp() + self.bundle = self.sql('SELECT content FROM context_source_versions').stdout.strip() + self.sql('''DELETE FROM context_documents; DELETE FROM context_result_sources; + DELETE FROM context_results; DELETE FROM context_attempts; + DELETE FROM context_source_versions; DELETE FROM context_sources;''') + + # Adopted reader tests need their original saved fixture; run them separately. + def retain(self, key='first', revision=0, bundle=None, catalog=2, request=4): + content = (self.bundle if bundle is None else json.dumps(bundle)).replace("'", "''") + return json.loads(self.sql(f"SELECT stream_binding_proposal.retain('{uid(1)}','{uid(1)}','{uid(request)}','{uid(5)}','{uid(catalog)}','{key}',{revision},'{content}'::jsonb)").stdout) + + def reconcile(self, key='first', actor=1): + return json.loads(self.sql(f"SELECT stream_binding_proposal.reconcile_retention('{uid(actor)}','{uid(1)}','{uid(4)}','{key}')").stdout) + + def test_writer_first_create_retry_reconcile(self): + receipt = self.retain() + self.assertEqual(receipt['revision'], 1) + self.assertEqual(self.retain(), receipt) + self.assertEqual(self.reconcile(), receipt) + self.assertEqual(self.sql('SELECT count(*) FROM context_attempts').stdout.strip(), '1') + self.assertEqual(self.sql('SELECT count(*) FROM context_source_versions').stdout.strip(), '1') + + def test_writer_changed_retry_and_revision_conflict_roll_back(self): + self.retain() + with self.assertRaisesRegex(AssertionError, 'command conflict'): + self.retain(revision=1) + with self.assertRaisesRegex(AssertionError, 'revision conflict'): + self.retain(key='other') + self.assertEqual(self.sql('SELECT count(*) FROM context_attempts').stdout.strip(), '1') + + def test_writer_invalid_bundle_rolls_back_all_records(self): + bad = json.loads(self.bundle); bad['inventedGrowth'] = 999 + self.assertIsNone(self.retain(bundle=bad)) + for table in ('context_sources','context_source_versions','context_attempts','context_results','context_result_sources','context_documents'): + self.assertEqual(self.sql(f'SELECT count(*) FROM {table}').stdout.strip(), '0') + + def test_writer_concurrent_first_create_one_cas_winner(self): + def save(key): + try: + return self.retain(key=key) + except AssertionError as exc: + self.assertIn('revision conflict', str(exc)); return 'conflict' + with ThreadPoolExecutor(max_workers=2) as pool: + values = list(pool.map(save, ['a','b'])) + self.assertEqual(sum(isinstance(x,dict) for x in values), 1) + self.assertEqual(values.count('conflict'), 1) + self.assertEqual(self.sql('SELECT count(*) FROM context_sources').stdout.strip(), '1') + + def test_writer_concurrent_same_key_replays_one_receipt(self): + with ThreadPoolExecutor(max_workers=2) as pool: + values = list(pool.map(lambda _: self.retain(), range(2))) + self.assertEqual(values[0],values[1]) + self.assertEqual(self.sql('SELECT count(*) FROM context_results').stdout.strip(), '1') + + def test_writer_new_key_deduplicates_version_and_stales_old_receipt(self): + first = self.retain() + second = self.retain(key='second', revision=1) + self.assertEqual(first['sourceVersionId'], second['sourceVersionId']) + self.assertNotEqual(first['resultId'], second['resultId']) + self.assertIsNone(self.reconcile()) + self.assertEqual(self.reconcile('second'), second) + self.assertEqual(self.sql('SELECT count(*) FROM context_source_versions').stdout.strip(), '1') + + def test_writer_customer_correction_and_removed_version_preserved(self): + receipt = self.retain() + self.sql("UPDATE context_results SET evidence_kind='customer_assertion'") + self.assertIsNone(self.retain(key='new', revision=1)) + self.assertEqual(self.sql('SELECT current_result_id FROM context_documents').stdout.strip(), receipt['resultId']) + self.assertEqual(self.sql('SELECT revision FROM context_documents').stdout.strip(), '1') + self.sql("UPDATE context_results SET evidence_kind='observation'; UPDATE context_source_versions SET removed_at=now()") + self.assertIsNone(self.retain(key='new', revision=1)) + self.assertIsNone(self.reconcile()) + + def test_writer_withdrawal_and_revocation_no_recreation(self): + self.retain() + self.sql("SELECT withdraw_context_source(owner_id,id) FROM context_sources") + self.assertIsNone(self.retain(key='new',revision=2)) + self.assertIsNone(self.reconcile()) + self.assertEqual(self.sql('SELECT count(*) FROM context_sources').stdout.strip(), '1') + self.setUp(); self.retain() + self.sql('DELETE FROM catalog_songs') + self.assertIsNone(self.reconcile()) + self.assertIsNone(self.retain(key='new',revision=1)) + self.assertEqual(self.sql('SELECT count(*) FROM context_results').stdout.strip(), '1') + + def test_writer_failed_replacement_restores_all_rows_and_document(self): + original = self.retain() + before = self.sql('SELECT row_to_json(d) FROM context_documents d').stdout + bad = json.loads(self.bundle); bad['inventedGrowth'] = 999 + self.assertIsNone(self.retain(key='bad', revision=1, bundle=bad)) + self.assertEqual(self.sql('SELECT row_to_json(d) FROM context_documents d').stdout, before) + self.assertEqual(self.reconcile(), original) + for table in ('context_sources','context_source_versions','context_attempts','context_results','context_result_sources','context_documents'): + self.assertEqual(self.sql(f'SELECT count(*) FROM {table}').stdout.strip(), '1') + + def test_writer_receipt_failure_rolls_back_atomic_save(self): + self.sql("""CREATE FUNCTION stream_binding_proposal.fail_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'Injected receipt failure'; END $$; + CREATE TRIGGER fail_receipt BEFORE UPDATE OF validation ON context_results + FOR EACH ROW EXECUTE FUNCTION stream_binding_proposal.fail_receipt();""") + try: + with self.assertRaisesRegex(AssertionError, 'Injected receipt failure'): + self.retain() + for table in ('context_sources','context_source_versions','context_attempts','context_results','context_result_sources','context_documents'): + self.assertEqual(self.sql(f'SELECT count(*) FROM {table}').stdout.strip(), '0') + finally: + self.sql('DROP TRIGGER fail_receipt ON context_results; DROP FUNCTION stream_binding_proposal.fail_receipt()') + + def test_writer_provider_history_switch_rolls_back(self): + self.retain() + self.sql("UPDATE catalog_stream_observations SET provider_recording_id='provider-b'") + changed=json.loads(self.bundle); changed['providerRecordingId']='provider-b' + changed['observations'][0]['provider_recording_id']='provider-b' + self.assertIsNone(self.retain(key='switch',revision=1,bundle=changed)) + self.assertEqual(self.sql('SELECT count(*) FROM context_source_versions').stdout.strip(), '1') + self.assertEqual(self.sql('SELECT revision FROM context_documents').stdout.strip(), '1') + + def test_writer_observation_work_budget_overflow(self): + self.sql("""INSERT INTO catalog_stream_runs(catalog_id,revision,scheduled_day,since,until) + SELECT catalog_id,revision,'1900-01-01'::date+i,'2026-10-24','2026-10-24' + FROM catalog_stream_tracking CROSS JOIN generate_series(1,4095) i; + INSERT INTO catalog_stream_observations + SELECT r.id,r.catalog_id,'USAAA2600001','2026-10-24',100,'provider-a',repeat('a',64),'2026-10-24T00:00:00Z' + FROM catalog_stream_runs r WHERE r.scheduled_day<'2020-01-01';""") + self.assertIsInstance(self.retain(),dict) + self.sql("""WITH added AS (INSERT INTO catalog_stream_runs(catalog_id,revision,scheduled_day,since,until) + SELECT catalog_id,revision,'1900-01-01'::date+4096,'2026-10-24','2026-10-24' FROM catalog_stream_tracking RETURNING *) + INSERT INTO catalog_stream_observations SELECT id,catalog_id,'USAAA2600001','2026-10-24',100,'provider-a',repeat('a',64),'2026-10-24T00:00:00Z' FROM added;""") + self.assertIsNone(self.reconcile()) + self.assertIsNone(self.retain(key='overflow',revision=1)) + self.assertEqual(self.sql('SELECT count(*) FROM context_results').stdout.strip(), '1') + + def test_writer_source_history_budget_prevents_new_and_retry_reads(self): + self.retain() + self.sql("""INSERT INTO context_source_versions(owner_id,source_id,fingerprint,content,retrieved_at) + SELECT owner_id,source_id,md5(i::text)||md5(i::text),content,retrieved_at + FROM context_source_versions CROSS JOIN generate_series(1,256) i;""") + self.assertIsNone(self.reconcile()) + self.assertIsNone(self.retain(key='overflow',revision=1)) + self.assertEqual(self.sql('SELECT count(*) FROM context_results').stdout.strip(), '1') + + def test_writer_read_committed_and_default_denial(self): + value=self.sql(f"BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT stream_binding_proposal.reconcile_retention('{uid(1)}','{uid(1)}','{uid(4)}','first'); ROLLBACK").stdout + self.assertIn('null',value.splitlines()) + self.assertEqual(self.sql("SELECT has_function_privilege('service_role','stream_binding_proposal.retain(uuid,uuid,uuid,uuid,uuid,text,bigint,jsonb)','EXECUTE')").stdout.strip(),'f') + + def test_writer_held_replacement_blocks_supported_mutators(self): + self.retain() + content=self.bundle.replace("'","''") + holder=subprocess.Popen(['psql','-h',self.tmp.name,'-p','55473','-d','postgres','-XAt','-v','ON_ERROR_STOP=1'], + stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + try: + holder.stdin.write(f"BEGIN; SELECT stream_binding_proposal.retain('{uid(1)}','{uid(1)}','{uid(4)}','{uid(5)}','{uid(2)}','held',1,'{content}'::jsonb);\n\\echo HELD\n") + holder.stdin.flush() + while True: + line=holder.stdout.readline() + if not line: self.fail('Holder failed: '+holder.stderr.read()) + if line.strip()=='HELD': break + for sql in ['DELETE FROM catalog_songs','DELETE FROM account_catalogs', + 'SELECT withdraw_context_source(owner_id,id) FROM context_sources', + f"SELECT commit_catalog_stream_track('{uid(7)}','USAAA2600001','{{\"state\":\"unavailable\"}}')"]: + with self.assertRaisesRegex(AssertionError,'statement timeout'): + self.sql("SET statement_timeout='200ms'; "+sql) + finally: + holder.communicate('COMMIT;\n',timeout=10) + self.assertEqual(holder.returncode,0) + self.assertEqual(self.reconcile('held')['revision'],2) + + + def second_catalog(self): + self.sql(f"""INSERT INTO catalogs VALUES ('{uid(20)}','Second synthetic catalog',now()); + INSERT INTO account_catalogs VALUES ('{uid(1)}','{uid(20)}'); + INSERT INTO catalog_songs VALUES ('USAAA2600001','{uid(20)}'); + INSERT INTO context_resources(id,provider,resource_kind,provider_id,canonical_url) + VALUES ('{uid(21)}','recoup','catalog','{uid(20)}','urn:recoup:catalog:{uid(20)}'); + INSERT INTO context_subjects(id,kind,catalog_id) VALUES ('{uid(22)}','catalog','{uid(20)}'); + INSERT INTO context_requests(id,owner_id,created_by,resource_id,idempotency_key,input_fingerprint,input,status,output) + VALUES ('{uid(23)}','{uid(1)}','{uid(1)}','{uid(21)}','second',repeat('a',64),'{{}}','partial','{{"subjectIds":["{uid(22)}"]}}'); + INSERT INTO context_request_catalog_members(request_id,owner_id,catalog_subject_id,recording_subject_id,catalog_id,song_isrc) + VALUES ('{uid(23)}','{uid(1)}','{uid(22)}','{uid(5)}','{uid(20)}','USAAA2600001'); + INSERT INTO catalog_stream_tracking(catalog_id,owner_id) VALUES ('{uid(20)}','{uid(1)}'); + INSERT INTO catalog_stream_runs(id,catalog_id,revision,scheduled_day,since,until) + SELECT '{uid(24)}',catalog_id,revision,'2026-10-25','2026-10-24','2026-10-25' FROM catalog_stream_tracking WHERE catalog_id='{uid(20)}'; + INSERT INTO catalog_stream_observations SELECT '{uid(24)}','{uid(20)}',isrc,date,streams,provider_recording_id,source_hash,retrieved_at + FROM catalog_stream_observations WHERE catalog_id='{uid(2)}'; + """) + bundle=json.loads(self.bundle); bundle['catalogId']=uid(20) + for row in bundle['observations']: + row['catalog_id']=uid(20); row['run_id']=uid(24) + return bundle + + def test_writer_cross_catalog_first_create_and_replace_cas(self): + other=self.second_catalog() + def save(args): + try: return self.retain(**args) + except AssertionError as exc: + self.assertIn('revision conflict',str(exc)); return 'conflict' + args=[{}, {'catalog':20,'request':23,'bundle':other}] + with ThreadPoolExecutor(max_workers=2) as pool: + values=list(pool.map(save,args)) + self.assertEqual(values.count('conflict'),1) + winner=next(x for x in values if isinstance(x,dict)) + target=args[1] if winner['catalogId']==uid(2) else args[0] + self.assertEqual(self.sql('SELECT count(*) FROM context_sources').stdout.strip(),'1') + replacement=self.retain(**target,key='replace',revision=1) + self.assertEqual(replacement['revision'],2) + with ThreadPoolExecutor(max_workers=2) as pool: + values=list(pool.map(save,[dict(x,key='race',revision=2) for x in args])) + self.assertEqual(values.count('conflict'),1) + self.assertEqual(self.sql('SELECT count(*) FROM context_documents').stdout.strip(),'1') + self.assertEqual(self.sql('SELECT revision FROM context_documents').stdout.strip(),'3') + self.assertEqual(self.sql('SELECT count(*) FROM context_results').stdout.strip(),'3') + + def test_writer_real_stream_correction_requires_explicit_new_bundle(self): + first=self.retain() + self.sql(f"""INSERT INTO catalog_stream_runs(id,catalog_id,revision,scheduled_day,since,until) + SELECT '{uid(13)}',catalog_id,revision,'2026-10-26','2026-10-24','2026-10-24' FROM catalog_stream_tracking; + SELECT commit_catalog_stream_track('{uid(13)}','USAAA2600001',jsonb_build_object( + 'state','complete','days',jsonb_build_array(jsonb_build_object('date','2026-10-24','streams',200)), + 'provider_recording_id','provider-a','source_hash',repeat('b',64),'retrieved_at','2026-10-26T00:00:00Z'));""") + self.assertIsNone(self.reconcile()) + self.assertIsNone(self.retain(key='old',revision=1)) + updated=json.loads(self.bundle) + updated['observations'][0].update(run_id=uid(13),streams=200,source_hash='b'*64,retrieved_at='2026-10-26T00:00:00+00:00') + updated['retrievalRange']={'earliest':'2026-10-26T00:00:00+00:00','latest':'2026-10-26T00:00:00+00:00'} + receipt=self.retain(key='corrected',revision=1,bundle=updated) + self.assertEqual(receipt['revision'],2) + self.assertNotEqual(receipt['sourceVersionId'],first['sourceVersionId']) + self.assertEqual(json.loads(self.sql(f"SELECT content FROM context_source_versions WHERE id='{first['sourceVersionId']}'").stdout),json.loads(self.bundle)) + + + def test_writer_history_byte_budget_refuses_before_any_write(self): + self.sql(f"""INSERT INTO catalog_stream_runs(id,catalog_id,revision,scheduled_day,since,until) + SELECT '{uid(13)}',catalog_id,revision,'2026-10-26','2026-10-24','2026-10-24' FROM catalog_stream_tracking; + INSERT INTO catalog_stream_observations SELECT '{uid(13)}',catalog_id,isrc,date,streams,provider_recording_id,repeat('a',8388609),retrieved_at-interval '1 day' FROM catalog_stream_observations;""") + self.assertIsNone(self.retain()) + self.assertEqual(self.sql('SELECT count(*) FROM context_sources').stdout.strip(),'0') + + def test_writer_post_insert_link_budget_rolls_back_receipt(self): + original=self.retain() + self.sql("""WITH extra AS (INSERT INTO context_results(owner_id,attempt_id,subject_id,topic,reuse_key,status,evidence_kind,normalized_response,coverage) + SELECT owner_id,attempt_id,subject_id,topic,md5(i::text)||md5(i::text),status,evidence_kind,normalized_response,coverage + FROM context_results CROSS JOIN generate_series(1,511) i RETURNING id,owner_id) + INSERT INTO context_result_sources SELECT extra.owner_id,extra.id,v.id FROM extra CROSS JOIN context_source_versions v;""") + self.assertEqual(self.reconcile(),original) + self.assertIsNone(self.retain(key='overflow',revision=1)) + self.assertEqual(self.sql('SELECT count(*) FROM context_result_sources').stdout.strip(),'512') + self.assertEqual(self.sql('SELECT revision FROM context_documents').stdout.strip(),'1') + self.assertEqual(self.sql('SELECT count(*) FROM context_attempts').stdout.strip(),'1') + + def test_writer_wrong_actor_reconcile_is_redacted(self): + self.retain() + self.assertIsNone(self.reconcile(actor=12)) + self.assertIsNone(self.reconcile(key='absent')) + self.setUp(); self.sql("UPDATE context_requests SET status='queued'") + self.assertIsNone(self.retain()) + + + +if __name__ == '__main__': + # Original tests run on their original fixture; writer tests use empty evidence. + suite = unittest.TestSuite([unittest.defaultTestLoader.loadTestsFromTestCase(Binding), + unittest.TestSuite(Writer(name) for name in dir(Writer) if name.startswith('test_writer_'))]) + result = unittest.TextTestRunner(verbosity=2).run(suite) + sys.exit(not result.wasSuccessful()) diff --git a/proposals/stream-retention-writer-2207/writer.sql b/proposals/stream-retention-writer-2207/writer.sql new file mode 100644 index 0000000..aeed79a --- /dev/null +++ b/proposals/stream-retention-writer-2207/writer.sql @@ -0,0 +1,93 @@ +-- Offline proposal only. Existing private schema remains denied to all application roles. +-- Receipt lives in the existing result.validation JSON, keyed through attempt.module. +create function stream_binding_proposal.reconcile_retention(p_actor uuid,p_owner uuid,p_request uuid,p_key text) +returns jsonb language plpgsql set search_path='' set timezone='UTC' as $$ +declare r public.context_results; receipt jsonb; checked jsonb; +begin + if p_key is null or p_key !~ '^[A-Za-z0-9_-]{1,100}$' then return 'null'::jsonb; end if; + select x.* into r from public.context_results x join public.context_attempts a + on a.id=x.attempt_id and a.owner_id=x.owner_id + where a.owner_id=p_owner and a.request_id=p_request and a.module='retained-binding:'||p_key + and a.attempt=1 and a.recipe_version='retained-luminate-daily-v1'; + if not found then return 'null'::jsonb; end if; + receipt:=r.validation->'retentionReceipt'; + checked:=stream_binding_proposal.read_binding(p_actor,p_owner,p_request,r.subject_id, + (receipt->>'catalogId')::uuid,r.id,(receipt->>'sourceVersionId')::uuid,(receipt->>'revision')::bigint); + if checked is null or checked='null'::jsonb then return 'null'::jsonb; end if; + return receipt; +exception when no_data_found or too_many_rows or insufficient_privilege or data_exception then + return 'null'::jsonb; +end $$; + +create function stream_binding_proposal.retain(p_actor uuid,p_owner uuid,p_request uuid,p_subject uuid,p_catalog uuid, + p_key text,p_revision bigint,p_evidence jsonb) +returns jsonb language plpgsql set search_path='' set timezone='UTC' as $$ +declare + isrc_value text; uri text; command_digest text; content_fingerprint text; + source_key uuid; version_key uuid; attempt_key uuid; result_key uuid; + doc public.context_documents; existing public.context_results; v public.context_source_versions; + receipt jsonb; checked jsonb; result_count integer; +begin + if p_key is null or p_key !~ '^[A-Za-z0-9_-]{1,100}$' or p_revision is null or p_revision<0 + or p_evidence is null or jsonb_typeof(p_evidence)<>'object' + or octet_length(convert_to(p_evidence::text,'UTF8'))>524288 or pg_column_size(p_evidence)>524288 + then raise insufficient_privilege; end if; + isrc_value:=stream_binding_proposal.lock_scope(p_actor,p_owner,p_request,p_subject,p_catalog); + if isrc_value is null then raise insufficient_privilege; end if; + if not stream_binding_proposal.within_budget(p_owner,p_catalog,p_subject,isrc_value,(p_evidence->>'since')::date,(p_evidence->>'until')::date) then raise insufficient_privilege; end if; + command_digest:=encode(sha256(convert_to(jsonb_build_array(p_owner,p_request,p_subject,p_catalog,p_key,p_revision,p_evidence)::text,'UTF8')),'hex'); + -- Tracking serialization also fences same-request/key retries for this catalog. + select r.* into existing from public.context_results r join public.context_attempts a + on a.id=r.attempt_id and a.owner_id=r.owner_id + where a.owner_id=p_owner and a.request_id=p_request and a.module='retained-binding:'||p_key and a.attempt=1; + if found then + if existing.reuse_key<>command_digest then raise exception 'Retention command conflict'; end if; + return stream_binding_proposal.reconcile_retention(p_actor,p_owner,p_request,p_key); + end if; + uri:='urn:recoup:retained-catalog-streams:'||p_catalog::text||':'||p_subject::text||':all_dsps:worldwide'; + select count(*) into result_count from public.context_sources where owner_id=p_owner and kind='provider_metadata' and source_url=uri; + if result_count>1 then raise insufficient_privilege; end if; + if result_count=1 then + select id into source_key from public.context_sources where owner_id=p_owner and kind='provider_metadata' + and source_url=uri and withdrawn_at is null for update; + if source_key is null then raise insufficient_privilege; end if; + else + insert into public.context_sources(owner_id,kind,source_url) values(p_owner,'provider_metadata',uri) returning id into source_key; + end if; + content_fingerprint:=encode(sha256(convert_to(p_evidence::text,'UTF8')),'hex'); + select * into v from public.context_source_versions where source_id=source_key and fingerprint=content_fingerprint for share; + if found then + if v.removed_at is not null or v.content is distinct from p_evidence then raise insufficient_privilege; end if; + version_key:=v.id; + else + insert into public.context_source_versions(owner_id,source_id,fingerprint,content,retrieved_at) + values(p_owner,source_key,content_fingerprint,p_evidence,(p_evidence->'retrievalRange'->>'latest')::timestamptz) returning id into version_key; + end if; + -- Source lock precedes document. The document uniqueness fences cross-catalog first creation. + insert into public.context_documents(owner_id,subject_id,topic) values(p_owner,p_subject,'catalog_retained_streams') + on conflict(owner_id,subject_id,topic) do nothing; + select * into strict doc from public.context_documents where owner_id=p_owner and subject_id=p_subject + and topic='catalog_retained_streams' for update; + if doc.revision<>p_revision then raise exception 'Retention revision conflict'; end if; + if exists(select 1 from public.context_results where id=doc.current_result_id and evidence_kind='customer_assertion') + then raise insufficient_privilege; end if; + insert into public.context_attempts(owner_id,request_id,module,attempt,status,provider,recipe_version,schema_version,started_at,finished_at) + values(p_owner,p_request,'retained-binding:'||p_key,1,'succeeded','luminate','retained-luminate-daily-v1','1',now(),now()) returning id into attempt_key; + insert into public.context_results(owner_id,attempt_id,subject_id,topic,reuse_key,status,evidence_kind,normalized_response,coverage) + values(p_owner,attempt_key,p_subject,'catalog_retained_streams',command_digest,'accepted','observation',p_evidence, + jsonb_build_object('extent',case when p_evidence->'absentDates'='[]'::jsonb and p_evidence->'nullDates'='[]'::jsonb then 'full' else 'partial' end, + 'scope','requested_recording_dates','derivedFieldsAuthorized',false)) returning id into result_key; + insert into public.context_result_sources(owner_id,result_id,source_version_id) values(p_owner,result_key,version_key); + if not public.accept_context_result(p_owner,doc.id,result_key,p_revision) then raise insufficient_privilege; end if; + checked:=stream_binding_proposal.read_binding(p_actor,p_owner,p_request,p_subject,p_catalog,result_key,version_key,p_revision+1); + if checked is null or checked='null'::jsonb then raise insufficient_privilege; end if; + receipt:=jsonb_build_object('ownerId',p_owner,'requestId',p_request,'subjectId',p_subject,'catalogId',p_catalog, + 'key',p_key,'commandDigest',command_digest,'resultId',result_key,'sourceVersionId',version_key,'revision',p_revision+1); + update public.context_results set validation=jsonb_build_object('retentionReceipt',receipt) where id=result_key; + return receipt; +exception when no_data_found or too_many_rows or insufficient_privilege or data_exception or not_null_violation then + -- The function subtransaction rolls every tentative write back on refusal. + return 'null'::jsonb; +end $$; +revoke all on function stream_binding_proposal.retain(uuid,uuid,uuid,uuid,uuid,text,bigint,jsonb), + stream_binding_proposal.reconcile_retention(uuid,uuid,uuid,text) from public,anon,authenticated,service_role;