From c240f5d2e833b757fe11ce9effb97d7676ebaf08 Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:56:47 -0400 Subject: [PATCH 1/5] Propose retained stream evidence binding with disposable lock proof --- .github/workflows/stream-binding-proposal.yml | 35 ++++ .../stream-evidence-binding-2207/PROTOCOL.md | 186 ++++++++++++++++++ .../stream-evidence-binding-2207/fixture.sql | 35 ++++ .../stream-evidence-binding-2207/proposal.sql | 104 ++++++++++ .../test_binding.py | 159 +++++++++++++++ 5 files changed, 519 insertions(+) create mode 100644 .github/workflows/stream-binding-proposal.yml create mode 100644 proposals/stream-evidence-binding-2207/PROTOCOL.md create mode 100644 proposals/stream-evidence-binding-2207/fixture.sql create mode 100644 proposals/stream-evidence-binding-2207/proposal.sql create mode 100644 proposals/stream-evidence-binding-2207/test_binding.py diff --git a/.github/workflows/stream-binding-proposal.yml b/.github/workflows/stream-binding-proposal.yml new file mode 100644 index 0000000..3619da2 --- /dev/null +++ b/.github/workflows/stream-binding-proposal.yml @@ -0,0 +1,35 @@ +name: Retained stream binding proposal (offline only) +on: + pull_request: + branches: [main] + paths: + - "proposals/stream-evidence-binding-2207/**" + - "tests/test_release_cases.py" + - "supabase/migrations/**" + - ".github/workflows/stream-binding-proposal.yml" +permissions: + contents: read +jobs: + disposable-postgres: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + postgres: [15, 17] + steps: + - uses: actions/checkout@v4 + timeout-minutes: 1 + - name: Install PostgreSQL + timeout-minutes: 12 + run: | + sudo apt-get update + sudo apt-get install -y postgresql-common + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get update + sudo apt-get install -y postgresql-${{ matrix.postgres }} + - name: Qualify retained evidence joins and locks + timeout-minutes: 3 + run: | + export PATH="/usr/lib/postgresql/${{ matrix.postgres }}/bin:$PATH" + python3 proposals/stream-evidence-binding-2207/test_binding.py diff --git a/proposals/stream-evidence-binding-2207/PROTOCOL.md b/proposals/stream-evidence-binding-2207/PROTOCOL.md new file mode 100644 index 0000000..1b6b930 --- /dev/null +++ b/proposals/stream-evidence-binding-2207/PROTOCOL.md @@ -0,0 +1,186 @@ +# Retained Luminate → Context evidence proposal (#2207) + +Status: source/design proposal, 2026-10-27. No migration, production writer, +service grant, provider call, ingestion service or durable adapter is implemented. +`proposal.sql` is an executable **read/lock qualification**, not an activation +script. The private proposal schema denies service/browser execution. Catalog +meaning review and root adoption remain required before runtime implementation. + +## Existing sources and exact scope + +Database base `6ae03a0cc71dc2ad3955594c9cfcb6156c3fb8b8`; API base +`c1298deb6bb981543bf442d26a32def889e60d71`. The relevant existing migrations are: + +- `20260920010000_context_foundation.sql`: sources, versions, attempts, accepted + results, current documents, `accept_context_result`, `withdraw_context_source`. +- `20260920020000_context_spotify_pipeline.sql`: metadata writer and ordinary + request document reader. The writer hardcodes Spotify; do not reuse it here. +- `20260922070000_context_catalog_entry.sql` and + `20260923080000_context_execution_records.sql`: owned catalog request, explicit + expansion edges and exact `resolve_context_catalog_member_recording`. +- `20261008160000_context_release_cases.sql`: current actor/workspace authorization. +- `20261009193000_catalog_daily_stream_tracking.sql` and + `20261010040000_catalog_stream_parent_lock_order.sql`: retained daily tuple, + latest-per-day ordering and current collection lock order. + +A global recording subject is not source authority. Require an existing expanded +member of an owned ready request, exact Recoup catalog resource and current +workspace/catalog/ISRC memberships. Never expand a request, infer a UUID, call a +provider or create a recording while resolving a selection. Expanded recordings +are deliberately outside `request.output.subjectIds`. + +## Minimal evidence representation + +Reuse the existing Context tables; add no observation mirror table or identifier +registry. The suggested source scope is **owner + catalog + recording**, with +`kind=provider_metadata` and server-generated locator +`urn:recoup:luminate-daily::`. It is a retained +evidence locator, not a provider URL or provider identity assertion. + +Use topic `luminate_daily_streams_v1:`, attempt provider `luminate`, +recipe `retained-luminate-daily-v1`, schema `1`, accepted result evidence kind +`observation`. The attempt records local retention work, not a new provider request +or a claim about historical provider costs. A recording in two catalogs has two +independent source/document scopes even in the same workspace. + +One exact source version contains the bounded snapshot; one result links exactly +that version. This preserves up to 62 daily observations without violating the +ObjectRef maximum of 32 source-version IDs. Suggested content/normalized result: + +```json +{ + "schemaVersion": 1, + "catalogId": "", + "subjectId": "", + "isrc": "USAAA2600001", + "metric": "luminate_all_dsp_worldwide_daily_streams", + "since": "2026-10-24", + "until": "2026-10-25", + "observations": [ + { + "catalog_id": "", + "isrc": "USAAA2600001", + "date": "2026-10-24", + "run_id": "", + "provider_recording_id": "provider-a", + "source_hash": "", + "retrieved_at": "2026-10-25T00:00:00+00:00", + "streams": 100 + } + ] +} +``` + +`since` inclusive / `until` exclusive, 1–62 completed UTC days; observations sorted +by date. Preserve all eight tuple fields, including nullable streams. Missing days +stay missing; null is unknown, never zero. Reject empty snapshots and mixed/empty +provider IDs. Run ID must join the same catalog. Latest selection exactly matches +Catalog's `retrieved_at DESC, run_id DESC`, without choosing a convenient older row. +The metric is all-DSP worldwide Luminate daily streams; no rights, royalties, +Spotify-only measure or aggregate catalog totals follow from this evidence. + +Use SHA256 of UTC-serialized database JSONB text for **internal version deduplication** +only. Do not confuse it with the retained `source_hash`, or promise equivalence to +JavaScript JSON serialization. Source-version `retrieved_at` records local retention; +each observation's provider retrieval time remains in content. + +Minimality tradeoff: one current document per owner/recording/catalog. A newly +accepted snapshot of another window also supersedes the prior current result; +old selections refuse rather than silently refresh. If parallel windows must stay +current, root/Catalog should instead adopt a window-qualified topic before building +the writer. That is a document-currentness choice, not a new evidence store. + +## Proposed retention transaction (not implemented by this proof) + +1. Trusted API supplies authenticated actor/owner, exact existing request, subject, + catalog, window, expected current revision and expected observed tuple set. + Validate strict shapes and byte bounds in the API; do not accept source content, + provider assertions or owner IDs directly from descriptor payloads. +2. Lock owner account → catalog → song (KEY SHARE), tracking row (UPDATE), actor + organization membership (SHARE), workspace/catalog then catalog/song memberships + (SHARE), request/expanded edge/resource/subjects (SHARE). Recheck the exact + resolver under these locks. Do not require collection `enabled=true` for retained + evidence. Tracking owner must match the selected owner. +3. Read latest retained observations in the exact bounded window and compare with + the submitted expected tuples. Corrections arriving since the surface snapshot + require explicit rebuild/reselection. Reject missing rows, mismatches, mixed + provider IDs and a changed provider identity relative to this source's retained + history; a provider identity conflict needs Catalog reconciliation, not a new UUID. +4. Under the tracking serialization lock, find **all** matching source locators, + including withdrawn sources. Refuse withdrawn, duplicate or ambiguous scope; + never recreate an active source to bypass withdrawal. Lock source before any + result/document. Initial creation and retry must serialize here. Existing generic + metadata/enrichment writers are not authorized for this reserved locator/topic. +5. Insert/reuse the exact immutable version, reject removed versions. Create a local + attempt/result with trusted content and exactly one lineage link. Use expected + document revision and `accept_context_result`; failure rolls back all inserts. + Identical current request/result/version replay can return the same receipt; + an old receipt cannot restore an old current result. Cross-request reuse must + retain the original request reference or create an explicitly new result; the + proof requires the attempt's request to match the authorized request. +6. Return owner/request/subject/catalog, topic, document revision, result ID and + exact version set. No provider request or collection is dispatched. A failed or + unknown acknowledgment requires exact receipt reconciliation, not paid retry. + +These steps still need executable writer/CAS/retry/rollback tests before activation. +The synthetic fixture deliberately inserts evidence directly to qualify the existing +relations; it is not that production writer and does not prove its idempotency. + +## Exact read and durable authorization + +`read_binding` demonstrates current actor/member checks and joins the exact result, +version, source, attempt and document revision. It checks snapshot content against +current retained tuples and returns a single redacted JSON null for absent, +inaccessible, unsupported, removed, mismatched or stale evidence. It never drops a +failed version or substitutes latest into an old selection. Operational SQL errors +remain errors; transport must not label them successful empty results. + +Future durable calls must hold their adopted session/chat/fence locks **before** +this source boundary. The source boundary locks source/version before result/document, +matching existing withdrawal. No callback from this boundary may acquire chat locks. +The proof refuses isolation levels other than READ COMMITTED, so a preexisting +repeatable-read snapshot cannot hide a correction committed before the tracking lock. +It locks existing run/observation rows against update/delete and exact lineage links; +the result UPDATE lock blocks new lineage links through their FK. +The proof holds tracking before membership and source, matching the supported stream +commit's parent order. It locks mutable request/resource/subject/edge participants +before the existing resolver. Current request output only needs the catalog subject. + +Withdrawal uses the existing `withdraw_context_source`: all versions in this +owner/catalog/recording scope become unavailable, results become withdrawn and current +document pointers clear with a revision increment. This proposed scope needs Catalog +acceptance. Collection disable retains evidence. Membership removal refuses future +reads without deleting global identities or retained history. Neither operation +proves removal from chat/export copies. No automatic source restoration is proposed. + +## Qualification and limits + +Run with a single PostgreSQL installation on PATH: + +```sh +python3 proposals/stream-evidence-binding-2207/test_binding.py +``` + +Fifteen tests passed on local PostgreSQL 15 and 17. The harness creates a new +Unix-socket-only cluster and applies the actual foundation, pipeline, catalog, +execution, access and stream migrations. Synthetic parents and evidence are explicit. +It never reads a hosted DSN. Tests cover exact expanded-member joins, wrong owner/ +actor/version/revision, missing memberships/edge/lineage, removed version, tuple +mutation, correction/null/provider-change staleness, explicit corrected version, +withdrawal and disable distinction. Real competing connections show that organization +revocation, catalog membership removal, withdrawal and tracking updates wait for an +authorized transaction. The inherited release-case lifecycle test also runs. + +Not proven: production grants/RLS delivery, generic-writer exclusion, arbitrary direct +SQL observation insertion/deletion races, all account/catalog cascade interleavings, +concurrent first retention, writer idempotency/rollback, durable chat composition, +strict API byte/schema validation or provider identity reconciliation. The tracking +lock serializes the existing `commit_catalog_stream_track`; it is not an enforcement +mechanism against direct table writers that bypass it. A runtime release must inventory +and constrain participating writers before claiming that atomic boundary. This is a +qualified proposal with explicit gaps, not production-ready authorization. + +Next: root obtains Catalog review of metric/provider conflict/withdrawal scope and +current-window choice; adopt or revise this representation; implement and independently +qualify the trusted retention writer and durable composition. No activation, migration, +provider/customer operation or grant follows from source merge alone. diff --git a/proposals/stream-evidence-binding-2207/fixture.sql b/proposals/stream-evidence-binding-2207/fixture.sql new file mode 100644 index 0000000..bea17b5 --- /dev/null +++ b/proposals/stream-evidence-binding-2207/fixture.sql @@ -0,0 +1,35 @@ +-- Synthetic fixture only. These inserts are NOT a proposed production writer. +TRUNCATE accounts, context_resources, songs, catalogs, account_catalogs, catalog_songs CASCADE; +INSERT INTO accounts(id,name) VALUES ('10000000-0000-4000-8000-000000000001','Synthetic owner'),('10000000-0000-4000-8000-000000000012','Other account'); +INSERT INTO catalogs VALUES ('10000000-0000-4000-8000-000000000002','Synthetic catalog',now()); +INSERT INTO songs VALUES ('USAAA2600001'); +INSERT INTO account_catalogs VALUES ('10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000002'); +INSERT INTO catalog_songs VALUES ('USAAA2600001','10000000-0000-4000-8000-000000000002'); +INSERT INTO context_resources(id,provider,resource_kind,provider_id,canonical_url) +VALUES ('10000000-0000-4000-8000-000000000003','recoup','catalog','10000000-0000-4000-8000-000000000002','urn:recoup:catalog:10000000-0000-4000-8000-000000000002'); +INSERT INTO context_subjects(id,kind,song_isrc) VALUES ('10000000-0000-4000-8000-000000000005','recording','USAAA2600001'); +INSERT INTO context_subjects(id,kind,catalog_id) VALUES ('10000000-0000-4000-8000-000000000006','catalog','10000000-0000-4000-8000-000000000002'); +INSERT INTO context_requests(id,owner_id,created_by,resource_id,idempotency_key,input_fingerprint,input,status,output) +VALUES ('10000000-0000-4000-8000-000000000004','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000003','fixture',repeat('a',64),'{}','partial','{"subjectIds":["10000000-0000-4000-8000-000000000006"]}'); +INSERT INTO context_request_catalog_members(request_id,owner_id,catalog_subject_id,recording_subject_id,catalog_id,song_isrc) +VALUES ('10000000-0000-4000-8000-000000000004','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000006','10000000-0000-4000-8000-000000000005','10000000-0000-4000-8000-000000000002','USAAA2600001'); +INSERT INTO catalog_stream_tracking(catalog_id,owner_id,enabled) VALUES ('10000000-0000-4000-8000-000000000002','10000000-0000-4000-8000-000000000001',true); +INSERT INTO catalog_stream_runs(id,catalog_id,revision,scheduled_day,since,until) +SELECT '10000000-0000-4000-8000-000000000007',catalog_id,revision,'2026-10-25','2026-10-24','2026-10-25' FROM catalog_stream_tracking; +INSERT INTO catalog_stream_observations VALUES ('10000000-0000-4000-8000-000000000007','10000000-0000-4000-8000-000000000002','USAAA2600001','2026-10-24',100,'provider-a','retained-hash','2026-10-25T00:00:00Z'); +INSERT INTO context_sources(id,owner_id,source_url,kind) +VALUES ('10000000-0000-4000-8000-000000000008','10000000-0000-4000-8000-000000000001','urn:recoup:luminate-daily:10000000-0000-4000-8000-000000000002:10000000-0000-4000-8000-000000000005','provider_metadata'); +INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) +SELECT '10000000-0000-4000-8000-000000000009','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000008',repeat('b',64),jsonb_build_object( +'schemaVersion',1,'catalogId','10000000-0000-4000-8000-000000000002','subjectId','10000000-0000-4000-8000-000000000005','isrc','USAAA2600001', +'metric','luminate_all_dsp_worldwide_daily_streams','since','2026-10-24','until','2026-10-25', +'observations',(SELECT jsonb_agg(to_jsonb(o) ORDER BY date) FROM catalog_stream_observations o)); +INSERT INTO context_attempts(id,owner_id,request_id,module,attempt,status,provider,recipe_version,schema_version) +VALUES ('10000000-0000-4000-8000-000000000010','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000004','retained-stream-binding',1,'succeeded','luminate','retained-luminate-daily-v1','1'); +INSERT INTO context_results(id,owner_id,attempt_id,subject_id,topic,reuse_key,status,evidence_kind,normalized_response,coverage) +SELECT '10000000-0000-4000-8000-000000000011','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000010','10000000-0000-4000-8000-000000000005','luminate_daily_streams_v1:10000000-0000-4000-8000-000000000002',repeat('c',64),'accepted','observation',content,'{"scope":"retained_rows_only"}' +FROM context_source_versions; +INSERT INTO context_result_sources VALUES ('10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000011','10000000-0000-4000-8000-000000000009'); +INSERT INTO context_documents(owner_id,subject_id,topic) +VALUES ('10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000005','luminate_daily_streams_v1:10000000-0000-4000-8000-000000000002'); +SELECT accept_context_result('10000000-0000-4000-8000-000000000001',id,'10000000-0000-4000-8000-000000000011',0) FROM context_documents; diff --git a/proposals/stream-evidence-binding-2207/proposal.sql b/proposals/stream-evidence-binding-2207/proposal.sql new file mode 100644 index 0000000..aab190c --- /dev/null +++ b/proposals/stream-evidence-binding-2207/proposal.sql @@ -0,0 +1,104 @@ +-- SOURCE-ONLY PROPOSAL. Not a migration, not a production authorization adapter. +-- No tables, runtime call sites, service grants, collection or implicit expansion. +create schema stream_binding_proposal; +revoke all on schema stream_binding_proposal from public,anon,authenticated,service_role; +create function stream_binding_proposal.read_binding( + p_actor uuid,p_owner uuid,p_request uuid,p_subject uuid,p_catalog uuid, + p_result uuid,p_version uuid,p_revision bigint +) returns jsonb language plpgsql set search_path='' set timezone='UTC' as $$ +declare + isrc_value text; resolved jsonb; payload jsonb; actual jsonb; source_key uuid; + first_day date; last_day date; topic_value text; source_uri text; +begin + if current_setting('transaction_isolation') <> 'read committed' then return 'null'::jsonb; end if; + -- Parent order follows commit_catalog_stream_track. Chat/session locks, if any, + -- must be acquired by a future durable caller before entering this boundary. + perform 1 from public.accounts where id=p_owner for key share; + if not found then return 'null'::jsonb; end if; + perform 1 from public.catalogs where id=p_catalog for key share; + if not found then return 'null'::jsonb; end if; + select song_isrc into isrc_value from public.context_subjects where id=p_subject and kind='recording'; + if isrc_value is null then return 'null'::jsonb; end if; + perform 1 from public.songs where isrc=isrc_value for key share; + perform 1 from public.catalog_stream_tracking where catalog_id=p_catalog and owner_id=p_owner for update; + if not found then return 'null'::jsonb; end if; + perform public.authorize_context_case_actor(p_actor,p_owner); + perform 1 from public.account_catalogs where account=p_owner and catalog=p_catalog for share; + if not found then return 'null'::jsonb; end if; + perform 1 from public.catalog_songs where catalog=p_catalog and song=isrc_value for share; + if not found then return 'null'::jsonb; end if; + perform 1 from public.context_requests where id=p_request and owner_id=p_owner for share; + perform 1 from public.context_request_catalog_members + where owner_id=p_owner and request_id=p_request and recording_subject_id=p_subject + and catalog_id=p_catalog and song_isrc=isrc_value for share; + if not found then return 'null'::jsonb; end if; + -- Lock all mutable identity participants before invoking the existing exact resolver. + perform 1 from public.context_resources r join public.context_requests q on q.resource_id=r.id + where q.id=p_request for share of r; + perform 1 from public.context_subjects s join public.context_request_catalog_members m + on s.id in (m.catalog_subject_id,m.recording_subject_id) + where m.owner_id=p_owner and m.request_id=p_request and m.recording_subject_id=p_subject + order by s.id for share of s; + resolved:=public.resolve_context_catalog_member_recording(p_owner,p_request,p_subject); + if resolved->>'isrc' is distinct from isrc_value then return 'null'::jsonb; end if; + topic_value:='luminate_daily_streams_v1:'||p_catalog::text; + source_uri:='urn:recoup:luminate-daily:'||p_catalog::text||':'||p_subject::text; + select s.id into source_key from public.context_sources s + join public.context_source_versions v on v.source_id=s.id and v.owner_id=s.owner_id + where s.owner_id=p_owner and s.source_url=source_uri and s.kind='provider_metadata' + and s.withdrawn_at is null and v.id=p_version and v.removed_at is null + for share of s,v; + if source_key is null then return 'null'::jsonb; end if; + -- Source before result/document agrees with withdraw_context_source. + select v.content into payload from public.context_results r + join public.context_attempts a on a.id=r.attempt_id and a.owner_id=r.owner_id + join public.context_result_sources rs on rs.result_id=r.id and rs.owner_id=r.owner_id + join public.context_source_versions v on v.id=rs.source_version_id and v.owner_id=rs.owner_id + join public.context_documents d on d.current_result_id=r.id and d.owner_id=r.owner_id + and d.subject_id=r.subject_id and d.topic=r.topic + where r.id=p_result and r.owner_id=p_owner and r.subject_id=p_subject and r.topic=topic_value + and r.status='accepted' and r.evidence_kind='observation' and a.request_id=p_request + and a.provider='luminate' and a.recipe_version='retained-luminate-daily-v1' + and a.status='succeeded' and a.schema_version='1' + and d.revision=p_revision and v.id=p_version and r.normalized_response=v.content + and (select count(*) from public.context_result_sources x where x.result_id=r.id)=1 + for update of r for share of a,d,rs; + if payload is null or payload->>'schemaVersion' is distinct from '1' + or payload->>'catalogId' is distinct from p_catalog::text + or payload->>'subjectId' is distinct from p_subject::text + or payload->>'isrc' is distinct from isrc_value + or payload->>'metric' is distinct from 'luminate_all_dsp_worldwide_daily_streams' + then return 'null'::jsonb; end if; + first_day:=(payload->>'since')::date; last_day:=(payload->>'until')::date; + if first_day is null or last_day is null or last_day-first_day not between 1 and 62 + then return 'null'::jsonb; end if; + -- Lock retained rows and their run parents against mutation/removal. This does + -- not prevent arbitrary direct inserts that bypass the tracking writer protocol. + perform 1 from public.catalog_stream_runs r where r.catalog_id=p_catalog + and exists(select 1 from public.catalog_stream_observations o where o.run_id=r.id + and o.catalog_id=p_catalog and o.isrc=isrc_value and o.date>=first_day and o.date=first_day and o.date=first_day and o.date'observations' + or (select count(distinct x->>'provider_recording_id') from jsonb_array_elements(actual) x)<>1 + or exists(select 1 from jsonb_array_elements(actual) x where x->>'provider_recording_id'='') + then return 'null'::jsonb; end if; + return jsonb_build_object('ownerId',p_owner,'subjectId',p_subject,'requestId',p_request, + 'catalogId',p_catalog,'resultId',p_result,'revision',p_revision,'topic',topic_value, + 'sourceVersionIds',jsonb_build_array(p_version),'evidence',payload); +exception when no_data_found or too_many_rows or insufficient_privilege or data_exception then + return 'null'::jsonb; +end $$; +revoke all on function stream_binding_proposal.read_binding(uuid,uuid,uuid,uuid,uuid,uuid,uuid,bigint) +from public,anon,authenticated,service_role; diff --git a/proposals/stream-evidence-binding-2207/test_binding.py b/proposals/stream-evidence-binding-2207/test_binding.py new file mode 100644 index 0000000..e281f8d --- /dev/null +++ b/proposals/stream-evidence-binding-2207/test_binding.py @@ -0,0 +1,159 @@ +"""Disposable proof against existing Context and Catalog migrations; never a hosted DSN.""" +import json +import os +import pathlib +import sys +import subprocess +import unittest + +# Canonical retained timestamp serialization; proposal reader also pins UTC. +os.environ['PGTZ'] = 'UTC' + +ROOT = pathlib.Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +import test_release_cases as harness + + +class Binding(harness.ReleaseCases): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.sql('''CREATE TABLE catalogs(id uuid PRIMARY KEY,name text,updated_at timestamptz); + CREATE TABLE account_catalogs(account uuid,catalog uuid); + CREATE TABLE catalog_songs(song text,catalog uuid);''') + for name in ('20260922070000_context_catalog_entry.sql', + '20260923080000_context_execution_records.sql', + '20261009193000_catalog_daily_stream_tracking.sql', + '20261010035000_catalog_stream_membership_locks.sql', + '20261010040000_catalog_stream_parent_lock_order.sql'): + cls.sql((ROOT / 'supabase/migrations' / name).read_text()) + cls.sql((pathlib.Path(__file__).parent / 'proposal.sql').read_text()) + + def setUp(self): + self.sql((pathlib.Path(__file__).parent / 'fixture.sql').read_text()) + + def call(self, actor=1, owner=1, version=9, revision=1): + return self.sql(f"SELECT stream_binding_proposal.read_binding('{uid(actor)}','{uid(owner)}','{uid(4)}','{uid(5)}','{uid(2)}','{uid(11)}','{uid(version)}',{revision})").stdout.strip() + + def test_exact_expanded_member(self): + value = json.loads(self.call()) + self.assertEqual(value['sourceVersionIds'], [uid(9)]) + self.assertEqual(value['subjectId'], uid(5)) + self.assertEqual(self.sql(f"SELECT output->'subjectIds' ? '{uid(5)}' FROM context_requests WHERE id='{uid(4)}'").stdout.strip(), 'f') + + def test_wrong_owner_actor_version_revision(self): + for args in ({'owner': 12}, {'actor': 12}, {'version': 12}, {'revision': 2}): + self.assertEqual(self.call(**args), 'null') + + def test_current_membership_and_expansion_required(self): + for table in ('account_catalogs', 'catalog_songs', 'context_request_catalog_members'): + self.setUp() + self.sql(f'DELETE FROM {table}') + self.assertEqual(self.call(), 'null') + + def test_withdrawal_clears_document_and_refuses(self): + self.sql(f"SELECT withdraw_context_source('{uid(1)}','{uid(8)}')") + self.assertEqual(self.call(), 'null') + self.assertEqual(self.sql('SELECT current_result_id IS NULL FROM context_documents').stdout.strip(), 't') + + def test_removed_version_and_missing_link(self): + self.sql('UPDATE context_source_versions SET removed_at=now()') + self.assertEqual(self.call(), 'null') + self.setUp() + self.sql('DELETE FROM context_result_sources') + self.assertEqual(self.call(), 'null') + + def test_correction_and_provider_conflict_refuse_old_binding(self): + for provider, streams in [('provider-a', 200), ('provider-b', 100), ('provider-a', 'NULL')]: + self.setUp() + self.sql(f"""INSERT INTO catalog_stream_runs(id,catalog_id,revision,scheduled_day,since,until) + SELECT '{uid(13)}',catalog_id,revision,'2026-10-26','2026-10-24','2026-10-25' FROM catalog_stream_tracking; + INSERT INTO catalog_stream_observations SELECT '{uid(13)}',catalog_id,isrc,date,{streams},'{provider}','correction',retrieved_at+interval '1 hour' FROM catalog_stream_observations;""") + self.assertEqual(self.call(), 'null') + + def test_tracking_disable_is_not_withdrawal(self): + self.sql('UPDATE catalog_stream_tracking SET enabled=false') + self.assertNotEqual(self.call(), 'null') + + def test_exact_tuple_tamper(self): + for column, value in [('source_hash', "'changed'"), ('streams', '0'), ('provider_recording_id', "'other'"), ('retrieved_at', "retrieved_at+interval '1 second'")]: + self.setUp() + self.sql(f'UPDATE catalog_stream_observations SET {column}={value}') + self.assertEqual(self.call(), 'null') + + def test_source_and_membership_locks(self): + statements = [ + 'DELETE FROM account_catalogs', + 'DELETE FROM catalog_songs', + f"SELECT withdraw_context_source('{uid(1)}','{uid(8)}')", + 'UPDATE catalog_stream_tracking SET enabled=false', + 'UPDATE catalog_stream_observations SET streams=999', + 'DELETE FROM context_result_sources', + 'DELETE FROM catalog_stream_runs', + f"INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(11)}','{uid(14)}')", + ] + self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content FROM context_source_versions") + self.hold_and_refuse(statements) + + def test_organization_revocation_lock(self): + self.sql(f"INSERT INTO account_organization_ids(account_id,organization_id) VALUES ('{uid(12)}','{uid(1)}')") + self.assertNotEqual(self.call(actor=12), 'null') + self.hold_and_refuse(['DELETE FROM account_organization_ids'], actor=12) + self.sql('DELETE FROM account_organization_ids') + self.assertEqual(self.call(actor=12), 'null') + + def hold_and_refuse(self, statements, actor=1): + holder = subprocess.Popen(['psql', '-h', self.tmp.name, '-p', '55473', '-d', 'postgres', + '-XAt', '-v', 'ON_ERROR_STOP=1'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + try: + holder.stdin.write(f"BEGIN; SELECT stream_binding_proposal.read_binding('{uid(actor)}','{uid(1)}','{uid(4)}','{uid(5)}','{uid(2)}','{uid(11)}','{uid(9)}',1);\n\\echo HELD\n") + holder.stdin.flush() + while True: + line = holder.stdout.readline() + if not line: + self.fail('Lock holder exited: ' + holder.stderr.read()) + if line.strip() == 'HELD': + break + for statement in statements: + with self.assertRaisesRegex(AssertionError, 'statement timeout'): + self.sql("SET statement_timeout='200ms'; " + statement) + finally: + holder.communicate('ROLLBACK;\n', timeout=10) + self.assertEqual(holder.returncode, 0) + + def test_current_result_and_extra_lineage_refused(self): + self.sql('UPDATE context_documents SET revision=revision+1') + self.assertEqual(self.call(), 'null') + self.setUp() + self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content FROM context_source_versions; INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(11)}','{uid(14)}')") + self.assertEqual(self.call(), 'null') + + def test_corrected_version_is_explicit_and_old_selection_stays_stale(self): + self.sql(f"""UPDATE catalog_stream_observations SET streams=200,source_hash='correction'; + INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) + SELECT '{uid(14)}',owner_id,source_id,repeat('e',64),jsonb_set(content,'{{observations}}', + (SELECT jsonb_agg(to_jsonb(o) ORDER BY date) FROM catalog_stream_observations o)) FROM context_source_versions; + INSERT INTO context_results(id,owner_id,attempt_id,subject_id,topic,reuse_key,status,evidence_kind,normalized_response,coverage) + SELECT '{uid(15)}',r.owner_id,r.attempt_id,r.subject_id,r.topic,repeat('f',64),r.status,r.evidence_kind,v.content,r.coverage + FROM context_results r CROSS JOIN context_source_versions v WHERE v.id='{uid(14)}'; + INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(15)}','{uid(14)}'); + SELECT accept_context_result('{uid(1)}',id,'{uid(15)}',1) FROM context_documents;""") + self.assertEqual(self.call(), 'null') + value = json.loads(self.sql(f"SELECT stream_binding_proposal.read_binding('{uid(1)}','{uid(1)}','{uid(4)}','{uid(5)}','{uid(2)}','{uid(15)}','{uid(14)}',2)").stdout) + self.assertEqual(value['evidence']['observations'][0]['streams'], 200) + + def test_repeatable_read_refused(self): + value = self.sql(f"BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT stream_binding_proposal.read_binding('{uid(1)}','{uid(1)}','{uid(4)}','{uid(5)}','{uid(2)}','{uid(11)}','{uid(9)}',1); ROLLBACK").stdout + self.assertIn('null', value.splitlines()) + + def test_no_service_execute(self): + self.assertEqual(self.sql("SELECT has_schema_privilege('service_role','stream_binding_proposal','USAGE')").stdout.strip(), 'f') + + +def uid(n): + return f'10000000-0000-4000-8000-{n:012d}' + + +if __name__ == '__main__': + unittest.main() From 8a286c280b6e3278eed005b609bc4784ea30f48c Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:57:43 -0400 Subject: [PATCH 2/5] Recheck lineage after waiting for result authority lock --- .../stream-evidence-binding-2207/PROTOCOL.md | 2 +- .../stream-evidence-binding-2207/proposal.sql | 6 +++- .../test_binding.py | 30 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/proposals/stream-evidence-binding-2207/PROTOCOL.md b/proposals/stream-evidence-binding-2207/PROTOCOL.md index 1b6b930..56c3995 100644 --- a/proposals/stream-evidence-binding-2207/PROTOCOL.md +++ b/proposals/stream-evidence-binding-2207/PROTOCOL.md @@ -161,7 +161,7 @@ Run with a single PostgreSQL installation on PATH: python3 proposals/stream-evidence-binding-2207/test_binding.py ``` -Fifteen tests passed on local PostgreSQL 15 and 17. The harness creates a new +Sixteen tests passed on local PostgreSQL 15 and 17. The harness creates a new Unix-socket-only cluster and applies the actual foundation, pipeline, catalog, execution, access and stream migrations. Synthetic parents and evidence are explicit. It never reads a hosted DSN. Tests cover exact expanded-member joins, wrong owner/ diff --git a/proposals/stream-evidence-binding-2207/proposal.sql b/proposals/stream-evidence-binding-2207/proposal.sql index aab190c..d5a3361 100644 --- a/proposals/stream-evidence-binding-2207/proposal.sql +++ b/proposals/stream-evidence-binding-2207/proposal.sql @@ -63,6 +63,10 @@ begin and d.revision=p_revision and v.id=p_version and r.normalized_response=v.content and (select count(*) from public.context_result_sources x where x.result_id=r.id)=1 for update of r for share of a,d,rs; + -- A link insertion may have held FK KEY SHARE before the result lock. Recheck + -- in a new READ COMMITTED statement after acquiring that lock. + if (select count(*) from public.context_result_sources where result_id=p_result)<>1 + then return 'null'::jsonb; end if; if payload is null or payload->>'schemaVersion' is distinct from '1' or payload->>'catalogId' is distinct from p_catalog::text or payload->>'subjectId' is distinct from p_subject::text @@ -70,7 +74,7 @@ begin or payload->>'metric' is distinct from 'luminate_all_dsp_worldwide_daily_streams' then return 'null'::jsonb; end if; first_day:=(payload->>'since')::date; last_day:=(payload->>'until')::date; - if first_day is null or last_day is null or last_day-first_day not between 1 and 62 + if first_day is null or last_day is null or last_day-first_day not between 1 and 62 or last_day>current_date then return 'null'::jsonb; end if; -- Lock retained rows and their run parents against mutation/removal. This does -- not prevent arbitrary direct inserts that bypass the tracking writer protocol. diff --git a/proposals/stream-evidence-binding-2207/test_binding.py b/proposals/stream-evidence-binding-2207/test_binding.py index e281f8d..f0180cb 100644 --- a/proposals/stream-evidence-binding-2207/test_binding.py +++ b/proposals/stream-evidence-binding-2207/test_binding.py @@ -4,6 +4,8 @@ import pathlib import sys import subprocess +import time +from concurrent.futures import ThreadPoolExecutor import unittest # Canonical retained timestamp serialization; proposal reader also pins UTC. @@ -143,6 +145,34 @@ def test_corrected_version_is_explicit_and_old_selection_stays_stale(self): value = json.loads(self.sql(f"SELECT stream_binding_proposal.read_binding('{uid(1)}','{uid(1)}','{uid(4)}','{uid(5)}','{uid(2)}','{uid(15)}','{uid(14)}',2)").stdout) self.assertEqual(value['evidence']['observations'][0]['streams'], 200) + def test_link_insert_before_reader_lock_is_rechecked(self): + self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content FROM context_source_versions") + holder = subprocess.Popen(['psql', '-h', self.tmp.name, '-p', '55473', '-d', 'postgres', + '-XAt', '-v', 'ON_ERROR_STOP=1'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + try: + holder.stdin.write(f"BEGIN; INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(11)}','{uid(14)}');\n\\echo HELD\n") + holder.stdin.flush() + while holder.stdout.readline().strip() != 'HELD': + if holder.poll() is not None: + self.fail('Link insertion holder exited') + with ThreadPoolExecutor(max_workers=1) as pool: + result = pool.submit(self.call) + deadline = time.monotonic() + 5 + saw_wait = False + while time.monotonic() < deadline: + waits = self.sql("SELECT count(*) FROM pg_stat_activity WHERE wait_event_type='Lock' AND query LIKE '%read_binding%'").stdout.strip() + if int(waits): + saw_wait = True + break + time.sleep(0.02) + holder.communicate('COMMIT;\n', timeout=10) + self.assertTrue(saw_wait, 'Reader never reached the conflicting result lock') + self.assertEqual(result.result(timeout=10), 'null') + finally: + if holder.poll() is None: + holder.communicate('ROLLBACK;\n', timeout=10) + def test_repeatable_read_refused(self): value = self.sql(f"BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT stream_binding_proposal.read_binding('{uid(1)}','{uid(1)}','{uid(4)}','{uid(5)}','{uid(2)}','{uid(11)}','{uid(9)}',1); ROLLBACK").stdout self.assertIn('null', value.splitlines()) From 24f2c41a71ad4c88a57882c2f83f427e53caf5eb Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:58:32 -0400 Subject: [PATCH 3/5] Refuse recreated withdrawn source scopes and state lock budget limits --- proposals/stream-evidence-binding-2207/PROTOCOL.md | 7 ++++++- proposals/stream-evidence-binding-2207/proposal.sql | 4 ++++ proposals/stream-evidence-binding-2207/test_binding.py | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/proposals/stream-evidence-binding-2207/PROTOCOL.md b/proposals/stream-evidence-binding-2207/PROTOCOL.md index 56c3995..215f809 100644 --- a/proposals/stream-evidence-binding-2207/PROTOCOL.md +++ b/proposals/stream-evidence-binding-2207/PROTOCOL.md @@ -161,7 +161,7 @@ Run with a single PostgreSQL installation on PATH: python3 proposals/stream-evidence-binding-2207/test_binding.py ``` -Sixteen tests passed on local PostgreSQL 15 and 17. The harness creates a new +Seventeen tests passed on local PostgreSQL 15 and 17. The harness creates a new Unix-socket-only cluster and applies the actual foundation, pipeline, catalog, execution, access and stream migrations. Synthetic parents and evidence are explicit. It never reads a hosted DSN. Tests cover exact expanded-member joins, wrong owner/ @@ -171,6 +171,11 @@ withdrawal and disable distinction. Real competing connections show that organiz revocation, catalog membership removal, withdrawal and tracking updates wait for an authorized transaction. The inherited release-case lifecycle test also runs. +The 62-day bound limits output dates, not correction history or the number of +rows scanned/locked: existing-row locks can cover many retained corrections per +date. This proof does not claim a bounded lock footprint. A runtime writer/read +needs a reviewed row/work budget and fail-closed overflow behavior. + Not proven: production grants/RLS delivery, generic-writer exclusion, arbitrary direct SQL observation insertion/deletion races, all account/catalog cascade interleavings, concurrent first retention, writer idempotency/rollback, durable chat composition, diff --git a/proposals/stream-evidence-binding-2207/proposal.sql b/proposals/stream-evidence-binding-2207/proposal.sql index d5a3361..2639170 100644 --- a/proposals/stream-evidence-binding-2207/proposal.sql +++ b/proposals/stream-evidence-binding-2207/proposal.sql @@ -43,6 +43,10 @@ begin if resolved->>'isrc' is distinct from isrc_value then return 'null'::jsonb; end if; topic_value:='luminate_daily_streams_v1:'||p_catalog::text; source_uri:='urn:recoup:luminate-daily:'||p_catalog::text||':'||p_subject::text; + -- A replacement active row cannot restore this withdrawn source scope. + if (select count(*) from public.context_sources where owner_id=p_owner + and kind='provider_metadata' and source_url=source_uri)<>1 + then return 'null'::jsonb; end if; select s.id into source_key from public.context_sources s join public.context_source_versions v on v.source_id=s.id and v.owner_id=s.owner_id where s.owner_id=p_owner and s.source_url=source_uri and s.kind='provider_metadata' diff --git a/proposals/stream-evidence-binding-2207/test_binding.py b/proposals/stream-evidence-binding-2207/test_binding.py index f0180cb..6de3005 100644 --- a/proposals/stream-evidence-binding-2207/test_binding.py +++ b/proposals/stream-evidence-binding-2207/test_binding.py @@ -177,6 +177,10 @@ def test_repeatable_read_refused(self): value = self.sql(f"BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT stream_binding_proposal.read_binding('{uid(1)}','{uid(1)}','{uid(4)}','{uid(5)}','{uid(2)}','{uid(11)}','{uid(9)}',1); ROLLBACK").stdout self.assertIn('null', value.splitlines()) + def test_active_replacement_cannot_bypass_withdrawn_scope(self): + self.sql(f"INSERT INTO context_sources(id,owner_id,source_url,kind,withdrawn_at) SELECT '{uid(14)}',owner_id,source_url,kind,now() FROM context_sources") + self.assertEqual(self.call(), 'null') + def test_no_service_execute(self): self.assertEqual(self.sql("SELECT has_schema_privilege('service_role','stream_binding_proposal','USAGE')").stdout.strip(), 'f') From 17d7622bfd080fc5d6b8b8ee25d86e0dd97252d7 Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:07:38 -0400 Subject: [PATCH 4/5] Align retained evidence proof with adopted Catalog source contract --- .../stream-evidence-binding-2207/PROTOCOL.md | 364 +++++++++--------- .../VECTOR-COVERAGE.md | 31 ++ .../stream-evidence-binding-2207/fixture.sql | 25 +- .../stream-evidence-binding-2207/proposal.sql | 53 ++- .../test_binding.py | 162 +++++++- 5 files changed, 423 insertions(+), 212 deletions(-) create mode 100644 proposals/stream-evidence-binding-2207/VECTOR-COVERAGE.md diff --git a/proposals/stream-evidence-binding-2207/PROTOCOL.md b/proposals/stream-evidence-binding-2207/PROTOCOL.md index 215f809..81ab557 100644 --- a/proposals/stream-evidence-binding-2207/PROTOCOL.md +++ b/proposals/stream-evidence-binding-2207/PROTOCOL.md @@ -1,191 +1,179 @@ -# Retained Luminate → Context evidence proposal (#2207) - -Status: source/design proposal, 2026-10-27. No migration, production writer, -service grant, provider call, ingestion service or durable adapter is implemented. -`proposal.sql` is an executable **read/lock qualification**, not an activation -script. The private proposal schema denies service/browser execution. Catalog -meaning review and root adoption remain required before runtime implementation. - -## Existing sources and exact scope - -Database base `6ae03a0cc71dc2ad3955594c9cfcb6156c3fb8b8`; API base -`c1298deb6bb981543bf442d26a32def889e60d71`. The relevant existing migrations are: - -- `20260920010000_context_foundation.sql`: sources, versions, attempts, accepted - results, current documents, `accept_context_result`, `withdraw_context_source`. -- `20260920020000_context_spotify_pipeline.sql`: metadata writer and ordinary - request document reader. The writer hardcodes Spotify; do not reuse it here. +# Retained Catalog stream → Context evidence proposal (#2207) + +Status: source/design qualification, 2026-10-27. Aligned to the Catalog source +contract adopted by root from API #1187, head +`b185ea9ded070cd6d621d2bf13843bf9df6e5a8f`, merge +`08f62b7474e52b2bd0114d2d2c62c642c17ef6bf`. +[Canonical Catalog contract](https://github.com/recoupable/api/blob/08f62b7474e52b2bd0114d2d2c62c642c17ef6bf/docs/contracts/catalog-stream-evidence-2207/README.md) +and its 31 synthetic vectors govern source meaning. Earlier versions of this +proposal's dynamic topic, different locator and 62-date limit are superseded. + +No migration, production writer, service grant, provider call, ingestion service or +live adapter. `proposal.sql` is a private, default-denied **evidence read/lock proof**. +Its fixture inserts are not a retention writer. It does not authorize derived sums, +growth or comparison fields: exact bundle equality is not a result-field validator. +Root and Catalog review the revised exact heads before any further implementation. + +## Existing source authority + +Database base `6ae03a0cc71dc2ad3955594c9cfcb6156c3fb8b8`; API inspection base +`c1298deb6bb981543bf442d26a32def889e60d71`, with adopted Catalog contract above. +Relevant Database migrations under `supabase/migrations/`: + +- `20260920010000_context_foundation.sql`: sources, versions, attempts, results, + current documents, `accept_context_result`, `withdraw_context_source`. +- `20260920020000_context_spotify_pipeline.sql` and current metadata replacement + `20260920040000_context_guests.sql`: Spotify-specific writer; do not reuse here. - `20260922070000_context_catalog_entry.sql` and - `20260923080000_context_execution_records.sql`: owned catalog request, explicit + `20260923080000_context_execution_records.sql`: owned catalog requests, explicit expansion edges and exact `resolve_context_catalog_member_recording`. -- `20261008160000_context_release_cases.sql`: current actor/workspace authorization. +- `20261008160000_context_release_cases.sql`: actor/workspace authorization. - `20261009193000_catalog_daily_stream_tracking.sql` and - `20261010040000_catalog_stream_parent_lock_order.sql`: retained daily tuple, - latest-per-day ordering and current collection lock order. - -A global recording subject is not source authority. Require an existing expanded -member of an owned ready request, exact Recoup catalog resource and current -workspace/catalog/ISRC memberships. Never expand a request, infer a UUID, call a -provider or create a recording while resolving a selection. Expanded recordings -are deliberately outside `request.output.subjectIds`. - -## Minimal evidence representation - -Reuse the existing Context tables; add no observation mirror table or identifier -registry. The suggested source scope is **owner + catalog + recording**, with -`kind=provider_metadata` and server-generated locator -`urn:recoup:luminate-daily::`. It is a retained -evidence locator, not a provider URL or provider identity assertion. - -Use topic `luminate_daily_streams_v1:`, attempt provider `luminate`, -recipe `retained-luminate-daily-v1`, schema `1`, accepted result evidence kind -`observation`. The attempt records local retention work, not a new provider request -or a claim about historical provider costs. A recording in two catalogs has two -independent source/document scopes even in the same workspace. - -One exact source version contains the bounded snapshot; one result links exactly -that version. This preserves up to 62 daily observations without violating the -ObjectRef maximum of 32 source-version IDs. Suggested content/normalized result: - -```json -{ - "schemaVersion": 1, - "catalogId": "", - "subjectId": "", - "isrc": "USAAA2600001", - "metric": "luminate_all_dsp_worldwide_daily_streams", - "since": "2026-10-24", - "until": "2026-10-25", - "observations": [ - { - "catalog_id": "", - "isrc": "USAAA2600001", - "date": "2026-10-24", - "run_id": "", - "provider_recording_id": "provider-a", - "source_hash": "", - "retrieved_at": "2026-10-25T00:00:00+00:00", - "streams": 100 - } - ] -} -``` - -`since` inclusive / `until` exclusive, 1–62 completed UTC days; observations sorted -by date. Preserve all eight tuple fields, including nullable streams. Missing days -stay missing; null is unknown, never zero. Reject empty snapshots and mixed/empty -provider IDs. Run ID must join the same catalog. Latest selection exactly matches -Catalog's `retrieved_at DESC, run_id DESC`, without choosing a convenient older row. -The metric is all-DSP worldwide Luminate daily streams; no rights, royalties, -Spotify-only measure or aggregate catalog totals follow from this evidence. - -Use SHA256 of UTC-serialized database JSONB text for **internal version deduplication** -only. Do not confuse it with the retained `source_hash`, or promise equivalence to -JavaScript JSON serialization. Source-version `retrieved_at` records local retention; -each observation's provider retrieval time remains in content. - -Minimality tradeoff: one current document per owner/recording/catalog. A newly -accepted snapshot of another window also supersedes the prior current result; -old selections refuse rather than silently refresh. If parallel windows must stay -current, root/Catalog should instead adopt a window-qualified topic before building -the writer. That is a document-currentness choice, not a new evidence store. - -## Proposed retention transaction (not implemented by this proof) - -1. Trusted API supplies authenticated actor/owner, exact existing request, subject, - catalog, window, expected current revision and expected observed tuple set. - Validate strict shapes and byte bounds in the API; do not accept source content, - provider assertions or owner IDs directly from descriptor payloads. -2. Lock owner account → catalog → song (KEY SHARE), tracking row (UPDATE), actor - organization membership (SHARE), workspace/catalog then catalog/song memberships - (SHARE), request/expanded edge/resource/subjects (SHARE). Recheck the exact - resolver under these locks. Do not require collection `enabled=true` for retained - evidence. Tracking owner must match the selected owner. -3. Read latest retained observations in the exact bounded window and compare with - the submitted expected tuples. Corrections arriving since the surface snapshot - require explicit rebuild/reselection. Reject missing rows, mismatches, mixed - provider IDs and a changed provider identity relative to this source's retained - history; a provider identity conflict needs Catalog reconciliation, not a new UUID. -4. Under the tracking serialization lock, find **all** matching source locators, - including withdrawn sources. Refuse withdrawn, duplicate or ambiguous scope; - never recreate an active source to bypass withdrawal. Lock source before any - result/document. Initial creation and retry must serialize here. Existing generic - metadata/enrichment writers are not authorized for this reserved locator/topic. -5. Insert/reuse the exact immutable version, reject removed versions. Create a local - attempt/result with trusted content and exactly one lineage link. Use expected - document revision and `accept_context_result`; failure rolls back all inserts. - Identical current request/result/version replay can return the same receipt; - an old receipt cannot restore an old current result. Cross-request reuse must - retain the original request reference or create an explicitly new result; the - proof requires the attempt's request to match the authorized request. -6. Return owner/request/subject/catalog, topic, document revision, result ID and - exact version set. No provider request or collection is dispatched. A failed or - unknown acknowledgment requires exact receipt reconciliation, not paid retry. - -These steps still need executable writer/CAS/retry/rollback tests before activation. -The synthetic fixture deliberately inserts evidence directly to qualify the existing -relations; it is not that production writer and does not prove its idempotency. - -## Exact read and durable authorization - -`read_binding` demonstrates current actor/member checks and joins the exact result, -version, source, attempt and document revision. It checks snapshot content against -current retained tuples and returns a single redacted JSON null for absent, -inaccessible, unsupported, removed, mismatched or stale evidence. It never drops a -failed version or substitutes latest into an old selection. Operational SQL errors -remain errors; transport must not label them successful empty results. - -Future durable calls must hold their adopted session/chat/fence locks **before** -this source boundary. The source boundary locks source/version before result/document, -matching existing withdrawal. No callback from this boundary may acquire chat locks. -The proof refuses isolation levels other than READ COMMITTED, so a preexisting -repeatable-read snapshot cannot hide a correction committed before the tracking lock. -It locks existing run/observation rows against update/delete and exact lineage links; -the result UPDATE lock blocks new lineage links through their FK. -The proof holds tracking before membership and source, matching the supported stream -commit's parent order. It locks mutable request/resource/subject/edge participants -before the existing resolver. Current request output only needs the catalog subject. - -Withdrawal uses the existing `withdraw_context_source`: all versions in this -owner/catalog/recording scope become unavailable, results become withdrawn and current -document pointers clear with a revision increment. This proposed scope needs Catalog -acceptance. Collection disable retains evidence. Membership removal refuses future -reads without deleting global identities or retained history. Neither operation -proves removal from chat/export copies. No automatic source restoration is proposed. - -## Qualification and limits - -Run with a single PostgreSQL installation on PATH: - -```sh -python3 proposals/stream-evidence-binding-2207/test_binding.py -``` - -Seventeen tests passed on local PostgreSQL 15 and 17. The harness creates a new -Unix-socket-only cluster and applies the actual foundation, pipeline, catalog, -execution, access and stream migrations. Synthetic parents and evidence are explicit. -It never reads a hosted DSN. Tests cover exact expanded-member joins, wrong owner/ -actor/version/revision, missing memberships/edge/lineage, removed version, tuple -mutation, correction/null/provider-change staleness, explicit corrected version, -withdrawal and disable distinction. Real competing connections show that organization -revocation, catalog membership removal, withdrawal and tracking updates wait for an -authorized transaction. The inherited release-case lifecycle test also runs. - -The 62-day bound limits output dates, not correction history or the number of -rows scanned/locked: existing-row locks can cover many retained corrections per -date. This proof does not claim a bounded lock footprint. A runtime writer/read -needs a reviewed row/work budget and fail-closed overflow behavior. - -Not proven: production grants/RLS delivery, generic-writer exclusion, arbitrary direct -SQL observation insertion/deletion races, all account/catalog cascade interleavings, -concurrent first retention, writer idempotency/rollback, durable chat composition, -strict API byte/schema validation or provider identity reconciliation. The tracking -lock serializes the existing `commit_catalog_stream_track`; it is not an enforcement -mechanism against direct table writers that bypass it. A runtime release must inventory -and constrain participating writers before claiming that atomic boundary. This is a -qualified proposal with explicit gaps, not production-ready authorization. - -Next: root obtains Catalog review of metric/provider conflict/withdrawal scope and -current-window choice; adopt or revise this representation; implement and independently -qualify the trusted retention writer and durable composition. No activation, migration, -provider/customer operation or grant follows from source merge alone. + `20261010040000_catalog_stream_parent_lock_order.sql`: retained rows, latest-day + ordering and supported collection lock order. + +A global recording UUID is not evidence authority. Require the exact existing +expanded member of an owned partial/completed request; current actor/workspace, +workspace/catalog and catalog/ISRC memberships; exact Recoup catalog resource and +recording binding. Expanded recordings are outside `request.output.subjectIds`. +The reader never expands, invents a UUID, collects or creates a recording. + +## Adopted series, topic and current-document behavior + +Use existing Context tables. One owned source per catalog/recording/metric scope: + +`urn:recoup:retained-catalog-streams:::all_dsps:worldwide` + +The source kind is `provider_metadata`; the locator is internal, not a provider URL +or fetch target. Provider recording ID is version content, never part of this key. +The stable topic is **`catalog_retained_streams`**. Existing document uniqueness +therefore gives **one current document per owner/recording/topic across catalogs +and windows**. A newly accepted catalog/window replaces via revision CAS, making +old bindings stale. No dynamic query topics or simultaneous-current-window promise. + +One version contains the full bounded tuple bundle, and one accepted observation +result links exactly that version. This keeps all 732 supported dates within one +of ObjectRef's 32 allowed version IDs. Attempt provider `luminate`, recipe +`retained-luminate-daily-v1`, schema `1` identify truthful local binding work. They +must not imply a provider request, paid collection or confirmed historical cost. + +The strict evidence content is reconstructed by the proof, with no additional keys: + +- `schemaVersion: 1`, `workspaceId`, `catalogId`, `subjectId`, `isrc`. +- `provider: luminate`, `metric: daily_streams`, `platform: all_dsps`, + `territory: worldwide`, `timezone: UTC`, `providerRecordingId`. +- `since` inclusive and `until` exclusive span two equal completed UTC periods, + each 1–366 days (2–732 total dates). `periods` matches Catalog's previous/current + start/end_exclusive, days and timezone shape. +- Sorted latest-per-date `observations`, with all eight fields: + `(catalog_id,isrc,date,run_id,provider_recording_id,source_hash,retrieved_at,streams)`. +- Sorted `absentDates` and `nullDates`, kept separate; no synthetic absent tuple. +- `retrievalRange: {earliest,latest}` from included retained observations. + +Latest selection is exactly `retrieved_at DESC, run_id DESC`. Select the full set, +then reject invalid lineage; never filter out a bad latest row to choose an older +or convenient provider identity. Every row must join its run's catalog and fall +within that run's **inclusive** since/until range. Collection runs remain at most +62 dates; a comparison bundle can span many runs. Final run status does not erase +earlier committed rows. No-row series refuses; zero and stored null remain distinct. +Mixed provider IDs, blank IDs, non-64-hex hashes and nonfinite timestamps refuse. +Accepted/stale/withdrawn results on the same source with a different retained +provider ID also refuse pending an owning identity-switch disposition. + +`context_source_versions.retrieved_at` equals the **maximum included retained +retrieval time**; the range remains in content. Binding time belongs in attempt/ +result provenance. Published/source-updated time remains unknown. No fake freshness. +A future writer computes SHA256 of normalized UTC PostgreSQL JSONB text for internal +version deduplication. It is distinct from Luminate's raw payload hash and the +JavaScript surface hash; never claim cross-runtime serialization equality. + +## Explicit size limits and field boundary + +Proof limits: evidence content at most **524,288 bytes (512 KiB)** in both UTF-8 +PostgreSQL JSONB text and `pg_column_size(JSONB)`; returned evidence envelope at most +**786,432 bytes (768 KiB)** by the same two checks. Reject overflow, never truncate +rows, versions or strings. The 732-date test spans 12 runs and fits these limits; +large provider text may still make a shorter bundle refuse. + +These evidence limits do not enlarge the existing durable command/source-part caps. +A future API must separately measure the actual outgoing serialized UTF-8 bytes, +and emit only a bounded field projection plus exact references into the descriptor. +Do not copy a large evidence bundle into a capped chat data part or saved controls. +The full evidence read envelope is not a durable mutation command. + +The proof requires `normalized_response = version.content` to qualify **evidence +only**. It does not qualify complete-period sums/growth, percentage rounding, +zero-baseline behavior, safe-total overflow or which descriptor field is supported. +Those need the strict writer/result-field validator using existing +`compareSpotifyDailyStreams` semantics before any field ObjectRef is enabled. +Incomplete evidence may be retained without authorizing invented numeric totals. + +## Proposed retention transaction (not implemented) + +1. Derive actor/owner from trusted authentication; strictly validate exact request, + subject, catalog, periods, expected revision and observed tuple set. The descriptor + cannot supply authoritative source content, provider assertions or owner IDs. +2. Hold durable session/chat/fence locks first when composing with a mutation. + Then owner account → catalog → song KEY SHARE; tracking UPDATE; actor membership, + workspace/catalog, catalog/song, request/edge/resource/subjects SHARE. Recheck the + exact expanded-member resolver. Require tracking owner, not enabled collection. +3. Load full current tuples under that boundary; validate all run/identity/shape/ + size and source-meaning checks. Compare expected tuples so a correction requires + explicit rebuild/reselection. Preserve all nulls/absences. Refuse provider switches + without an adopted source-owner disposition. Preserve explicit customer-assertion + corrections; no provider projection silently overwrites them. +4. Find all matching source locators, including withdrawn history, under tracking + serialization. Refuse withdrawn/duplicate/ambiguous scope; never recreate an + active source to evade withdrawal. Lock source before result/document. Reserve + this locator/topic against generic writer paths before activation. +5. Insert/reuse exact immutable version with retained retrieval max. Create a truthful + local attempt/result and exactly one lineage link. Validate derived fields before + field authorization. Use expected document revision and `accept_context_result`; + failure rolls back all inserts. Identical current replay can return its exact + receipt; an old receipt cannot restore an old current result. Preserve exact + originating request on reuse, or create an explicitly new result. +6. Return exact result/topic/revision/version receipt. Unknown acknowledgment requires + receipt reconciliation, not provider retry or implicit collection. + +Concurrent first creation, CAS/retry/rollback and receipt behavior are not implemented +or proven by direct fixture inserts. They remain required before runtime work. + +## Read/lock qualification and withdrawal + +`read_binding` requires READ COMMITTED. Otherwise an earlier repeatable-read snapshot +could miss a correction committed before the tracking lock. It authorizes exact +actor/member/request/subject and current result/revision/source/version, reconstructs +the full evidence bundle and compares retained tuples. It returns redacted JSON null +for unavailable/stale/malformed evidence; operational SQL errors remain errors. + +Source/version locks precede result/document, matching withdrawal. Result UPDATE +blocks new lineage FK links; existing links are SHARE locked. A new statement +rechecks lineage count after result-lock acquisition, covering an insertion committed +while the reader waited. Existing run/observation rows are SHARE locked against +update/delete. The supported stream commit serializes on tracking. No source→chat +callback or inverted chat lock acquisition is introduced. + +Withdrawal uses existing `withdraw_context_source`: all versions/results for that +owned series become unavailable, current pointers clear and revisions advance. +Historical withdrawn locator rows also prevent a replacement active source passing +the reader. Collection disable is not withdrawal. Membership revocation removes +current access without deleting retained rows/global identity. No provider retention +policy, automatic republication or erasure of prior chat/export copies is implied. + +## Qualification, limitations and next action + +Run `python3 proposals/stream-evidence-binding-2207/test_binding.py` with PostgreSQL +on PATH. It creates a disposable Unix-socket-only cluster, applies actual existing +migrations and uses explicitly synthetic parents/evidence. No hosted DSN is read. +The revised suite has 25 tests; see `VECTOR-COVERAGE.md` for Catalog vector mapping. + +The 732-date cap bounds output dates, **not retained correction history or lock +footprint**. Arbitrary direct inserts bypassing tracking are not serialized by this +proof. Runtime work needs participating-writer enforcement, a row/work budget and +fail-closed overflow behavior. No production grants/RLS delivery, all cascade +interleavings, production writer idempotency/CAS/rollback, actual API transport byte +validation, derived field validation or durable chat composition is qualified. + +Next: Catalog independently reviews alignment to adopted #1187; root reviews revised +exact heads and fresh CI. Only after exact protocol adoption may the executable +retention writer be built and qualified. Source merge alone activates nothing. diff --git a/proposals/stream-evidence-binding-2207/VECTOR-COVERAGE.md b/proposals/stream-evidence-binding-2207/VECTOR-COVERAGE.md new file mode 100644 index 0000000..28e33e9 --- /dev/null +++ b/proposals/stream-evidence-binding-2207/VECTOR-COVERAGE.md @@ -0,0 +1,31 @@ +# Catalog vector port map + +Source: API #1187 `b185ea9ded070cd6d621d2bf13843bf9df6e5a8f`, adopted merge +`08f62b7474e52b2bd0114d2d2c62c642c17ef6bf`, +[all 31 vectors](https://github.com/recoupable/api/blob/08f62b7474e52b2bd0114d2d2c62c642c17ef6bf/docs/contracts/catalog-stream-evidence-2207/vectors.json). +Every live mapping remains unsupported. These ports qualify evidence only; they +never claim that the vectors' comparison outputs or production intake are shipped. + +| Catalog vector IDs | Executable proof / remaining boundary | +| ------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| exact_bound_bundle; disabled_tracking_retains_evidence | Exact expanded member, stable source/version/result; disable remains readable. | +| known_zero_baseline; null_correction_bound; absent_date_bound; all_stored_nulls | Null/zero/missing evidence kept distinct; comparison arithmetic remains unqualified. | +| new_correction_bound; old_version_after_correction; correction_not_yet_context_bound | Explicit corrected version accepted with existing CAS; old tuple/version refuses. | +| mixed_provider_ids; all_dates_changed_provider | Full matching tuple set refuses mixed identity; new result differing from earlier accepted source identity refuses. | +| workspace_revoked; member_removed; unresolved_recording | Actor/membership/expanded-edge refusal, including real competing revocation locks. | +| cross_owner_version; unrelated_metadata_version; forged_hash_only | Wrong owner/version, missing/extra lineage and exact source scope refuse. UUIDs remain stored references, never hashes. | +| unsupported_field_relabel | Wrong metric/scope and extra content refuse. No comparison/financial field authorization exists. | +| source_withdrawn; version_removed; attempted_recreation_after_withdrawal | Existing withdrawal clears pointer; removed version and active replacement with withdrawn locator history refuse. | +| different_current_window | New window replaces one current document; old binding stale. Additional cross-catalog replacement proof enforces same stable topic/document. | +| unstable_chunked_read; arbitrary_caller_tuple | Exact retained tuple mismatch, malformed run/date lineage and bad retained fields refuse. Direct phantom inserts remain outside cooperative writer proof. | +| result_not_accepted | Stale result refuses. | +| customer_correction_blocks_refresh | Reader refuses customer_assertion as provider observation; preservation by the future retention writer is not proven. | +| entirely_absent | Reader requires at least one retained row; broader no-row intake behavior remains a writer obligation. | +| identical_collection_no_new_rows | Unchanged retained tuples stay identical by construction; production dedup/replay not implemented. | +| late_writer_cas_lost | Existing CAS used by fixture acceptance; concurrent production retention writer not implemented. | +| unsafe_retained_count | Existing table CHECK constrains rows; derived safe-total overflow and full ingestion validator remain unqualified. | +| storage_failure | Operational errors propagate; real transport failure behavior is not implemented by this SQL-only proof. | + +Additional executable boundaries: 12-run/732-date snapshot, >732 refusal, retained +retrieval max, malformed hash/provider/timestamp/run range, multi-byte UTF-8 overflow, +READ COMMITTED requirement, mutation locks and reverse-order lineage insertion. diff --git a/proposals/stream-evidence-binding-2207/fixture.sql b/proposals/stream-evidence-binding-2207/fixture.sql index bea17b5..a6453c3 100644 --- a/proposals/stream-evidence-binding-2207/fixture.sql +++ b/proposals/stream-evidence-binding-2207/fixture.sql @@ -16,20 +16,27 @@ VALUES ('10000000-0000-4000-8000-000000000004','10000000-0000-4000-8000-00000000 INSERT INTO catalog_stream_tracking(catalog_id,owner_id,enabled) VALUES ('10000000-0000-4000-8000-000000000002','10000000-0000-4000-8000-000000000001',true); INSERT INTO catalog_stream_runs(id,catalog_id,revision,scheduled_day,since,until) SELECT '10000000-0000-4000-8000-000000000007',catalog_id,revision,'2026-10-25','2026-10-24','2026-10-25' FROM catalog_stream_tracking; -INSERT INTO catalog_stream_observations VALUES ('10000000-0000-4000-8000-000000000007','10000000-0000-4000-8000-000000000002','USAAA2600001','2026-10-24',100,'provider-a','retained-hash','2026-10-25T00:00:00Z'); +INSERT INTO catalog_stream_observations VALUES ('10000000-0000-4000-8000-000000000007','10000000-0000-4000-8000-000000000002','USAAA2600001','2026-10-24',100,'provider-a',repeat('a',64),'2026-10-25T00:00:00Z'); INSERT INTO context_sources(id,owner_id,source_url,kind) -VALUES ('10000000-0000-4000-8000-000000000008','10000000-0000-4000-8000-000000000001','urn:recoup:luminate-daily:10000000-0000-4000-8000-000000000002:10000000-0000-4000-8000-000000000005','provider_metadata'); -INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) -SELECT '10000000-0000-4000-8000-000000000009','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000008',repeat('b',64),jsonb_build_object( -'schemaVersion',1,'catalogId','10000000-0000-4000-8000-000000000002','subjectId','10000000-0000-4000-8000-000000000005','isrc','USAAA2600001', -'metric','luminate_all_dsp_worldwide_daily_streams','since','2026-10-24','until','2026-10-25', -'observations',(SELECT jsonb_agg(to_jsonb(o) ORDER BY date) FROM catalog_stream_observations o)); +VALUES ('10000000-0000-4000-8000-000000000008','10000000-0000-4000-8000-000000000001','urn:recoup:retained-catalog-streams:10000000-0000-4000-8000-000000000002:10000000-0000-4000-8000-000000000005:all_dsps:worldwide','provider_metadata'); +INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content,retrieved_at) +SELECT '10000000-0000-4000-8000-000000000009','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000008',repeat('b',64), +jsonb_build_object('schemaVersion',1,'workspaceId','10000000-0000-4000-8000-000000000001', +'catalogId','10000000-0000-4000-8000-000000000002','subjectId','10000000-0000-4000-8000-000000000005', +'isrc','USAAA2600001','provider','luminate','metric','daily_streams','platform','all_dsps','territory','worldwide','timezone','UTC', +'since','2026-10-23','until','2026-10-25', +'periods',jsonb_build_object('previous',jsonb_build_object('start','2026-10-23','end_exclusive','2026-10-24'), +'current',jsonb_build_object('start','2026-10-24','end_exclusive','2026-10-25'),'days',1,'timezone','UTC'), +'providerRecordingId','provider-a','observations',(SELECT jsonb_agg(to_jsonb(o) ORDER BY date) FROM catalog_stream_observations o), +'absentDates',jsonb_build_array('2026-10-23'),'nullDates','[]'::jsonb, +'retrievalRange',jsonb_build_object('earliest','2026-10-25T00:00:00+00:00','latest','2026-10-25T00:00:00+00:00')), +'2026-10-25T00:00:00Z'; INSERT INTO context_attempts(id,owner_id,request_id,module,attempt,status,provider,recipe_version,schema_version) VALUES ('10000000-0000-4000-8000-000000000010','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000004','retained-stream-binding',1,'succeeded','luminate','retained-luminate-daily-v1','1'); INSERT INTO context_results(id,owner_id,attempt_id,subject_id,topic,reuse_key,status,evidence_kind,normalized_response,coverage) -SELECT '10000000-0000-4000-8000-000000000011','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000010','10000000-0000-4000-8000-000000000005','luminate_daily_streams_v1:10000000-0000-4000-8000-000000000002',repeat('c',64),'accepted','observation',content,'{"scope":"retained_rows_only"}' +SELECT '10000000-0000-4000-8000-000000000011','10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000010','10000000-0000-4000-8000-000000000005','catalog_retained_streams',repeat('c',64),'accepted','observation',content,'{"scope":"retained_rows_only"}' FROM context_source_versions; INSERT INTO context_result_sources VALUES ('10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000011','10000000-0000-4000-8000-000000000009'); INSERT INTO context_documents(owner_id,subject_id,topic) -VALUES ('10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000005','luminate_daily_streams_v1:10000000-0000-4000-8000-000000000002'); +VALUES ('10000000-0000-4000-8000-000000000001','10000000-0000-4000-8000-000000000005','catalog_retained_streams'); SELECT accept_context_result('10000000-0000-4000-8000-000000000001',id,'10000000-0000-4000-8000-000000000011',0) FROM context_documents; diff --git a/proposals/stream-evidence-binding-2207/proposal.sql b/proposals/stream-evidence-binding-2207/proposal.sql index 2639170..0fa4a1f 100644 --- a/proposals/stream-evidence-binding-2207/proposal.sql +++ b/proposals/stream-evidence-binding-2207/proposal.sql @@ -9,6 +9,8 @@ create function stream_binding_proposal.read_binding( declare isrc_value text; resolved jsonb; payload jsonb; actual jsonb; source_key uuid; first_day date; last_day date; topic_value text; source_uri text; + midpoint date; days_count integer; provider_id text; expected jsonb; envelope jsonb; + absent_days jsonb; null_days jsonb; earliest timestamptz; latest timestamptz; version_time timestamptz; begin if current_setting('transaction_isolation') <> 'read committed' then return 'null'::jsonb; end if; -- Parent order follows commit_catalog_stream_track. Chat/session locks, if any, @@ -41,8 +43,8 @@ begin order by s.id for share of s; resolved:=public.resolve_context_catalog_member_recording(p_owner,p_request,p_subject); if resolved->>'isrc' is distinct from isrc_value then return 'null'::jsonb; end if; - topic_value:='luminate_daily_streams_v1:'||p_catalog::text; - source_uri:='urn:recoup:luminate-daily:'||p_catalog::text||':'||p_subject::text; + topic_value:='catalog_retained_streams'; + source_uri:='urn:recoup:retained-catalog-streams:'||p_catalog::text||':'||p_subject::text||':all_dsps:worldwide'; -- A replacement active row cannot restore this withdrawn source scope. if (select count(*) from public.context_sources where owner_id=p_owner and kind='provider_metadata' and source_url=source_uri)<>1 @@ -54,7 +56,7 @@ begin for share of s,v; if source_key is null then return 'null'::jsonb; end if; -- Source before result/document agrees with withdraw_context_source. - select v.content into payload from public.context_results r + select v.content,v.retrieved_at into payload,version_time from public.context_results r join public.context_attempts a on a.id=r.attempt_id and a.owner_id=r.owner_id join public.context_result_sources rs on rs.result_id=r.id and rs.owner_id=r.owner_id join public.context_source_versions v on v.id=rs.source_version_id and v.owner_id=rs.owner_id @@ -75,11 +77,13 @@ begin or payload->>'catalogId' is distinct from p_catalog::text or payload->>'subjectId' is distinct from p_subject::text or payload->>'isrc' is distinct from isrc_value - or payload->>'metric' is distinct from 'luminate_all_dsp_worldwide_daily_streams' + or payload->>'metric' is distinct from 'daily_streams' + or octet_length(convert_to(payload::text,'UTF8'))>524288 or pg_column_size(payload)>524288 then return 'null'::jsonb; end if; first_day:=(payload->>'since')::date; last_day:=(payload->>'until')::date; - if first_day is null or last_day is null or last_day-first_day not between 1 and 62 or last_day>current_date + if first_day is null or last_day is null or last_day-first_day not between 2 and 732 or (last_day-first_day)%2<>0 or last_day>current_date then return 'null'::jsonb; end if; + days_count:=(last_day-first_day)/2; midpoint:=first_day+days_count; -- Lock retained rows and their run parents against mutation/removal. This does -- not prevent arbitrary direct inserts that bypass the tracking writer protocol. perform 1 from public.catalog_stream_runs r where r.catalog_id=p_catalog @@ -94,17 +98,50 @@ begin select distinct on (o.date) o.catalog_id,o.isrc,o.date,o.run_id,o.provider_recording_id, o.source_hash,o.retrieved_at,o.streams from public.catalog_stream_observations o - join public.catalog_stream_runs r on r.id=o.run_id and r.catalog_id=o.catalog_id where o.catalog_id=p_catalog and o.isrc=isrc_value and o.date>=first_day and o.date'observations' or (select count(distinct x->>'provider_recording_id') from jsonb_array_elements(actual) x)<>1 - or exists(select 1 from jsonb_array_elements(actual) x where x->>'provider_recording_id'='') + or exists(select 1 from jsonb_array_elements(actual) x where (x->>'provider_recording_id') !~ '[^[:space:]]' + or (x->>'source_hash') !~ '^[0-9a-fA-F]{64}$' + or not isfinite((x->>'retrieved_at')::timestamptz) + or not exists(select 1 from public.catalog_stream_runs r where r.id=(x->>'run_id')::uuid + and r.catalog_id=p_catalog and (x->>'date')::date between r.since and r.until)) then return 'null'::jsonb; end if; - return jsonb_build_object('ownerId',p_owner,'subjectId',p_subject,'requestId',p_request, + provider_id:=actual->0->>'provider_recording_id'; + -- An entirely replaced identity also needs an owning reconciliation disposition. + if exists(select 1 from public.context_source_versions v + join public.context_result_sources rs on rs.source_version_id=v.id and rs.owner_id=v.owner_id + join public.context_results r on r.id=rs.result_id and r.owner_id=rs.owner_id + where v.owner_id=p_owner and v.source_id=source_key and r.status in ('accepted','stale','withdrawn') + and r.subject_id=p_subject and r.topic=topic_value + and v.content->>'providerRecordingId' is distinct from provider_id) + then return 'null'::jsonb; end if; + select min((x->>'retrieved_at')::timestamptz),max((x->>'retrieved_at')::timestamptz) + into earliest,latest from jsonb_array_elements(actual) x; + select coalesce(jsonb_agg(d::date::text order by d),'[]'::jsonb) into absent_days + from generate_series(first_day::timestamp,(last_day-1)::timestamp,interval '1 day') d + where not exists(select 1 from jsonb_array_elements(actual) x where x->>'date'=d::date::text); + select coalesce(jsonb_agg(x->>'date' order by x->>'date'),'[]'::jsonb) into null_days + from jsonb_array_elements(actual) x where x->'streams'='null'::jsonb; + expected:=jsonb_build_object('schemaVersion',1,'workspaceId',p_owner,'catalogId',p_catalog, + 'subjectId',p_subject,'isrc',isrc_value,'provider','luminate','metric','daily_streams', + 'platform','all_dsps','territory','worldwide','timezone','UTC','since',first_day,'until',last_day, + 'periods',jsonb_build_object('previous',jsonb_build_object('start',first_day,'end_exclusive',midpoint), + 'current',jsonb_build_object('start',midpoint,'end_exclusive',last_day),'days',days_count,'timezone','UTC'), + 'providerRecordingId',provider_id,'observations',actual,'absentDates',absent_days,'nullDates',null_days, + 'retrievalRange',jsonb_build_object('earliest',earliest,'latest',latest)); + -- Exact reconstruction rejects extra/malformed fields, wrong scope, missing/null + -- relabeling, and invented freshness. This is evidence-only, not derived totals. + if payload is distinct from expected or version_time is distinct from latest + then return 'null'::jsonb; end if; + envelope:=jsonb_build_object('ownerId',p_owner,'subjectId',p_subject,'requestId',p_request, 'catalogId',p_catalog,'resultId',p_result,'revision',p_revision,'topic',topic_value, 'sourceVersionIds',jsonb_build_array(p_version),'evidence',payload); + if octet_length(convert_to(envelope::text,'UTF8'))>786432 or pg_column_size(envelope)>786432 + then return 'null'::jsonb; end if; + return envelope; exception when no_data_found or too_many_rows or insufficient_privilege or data_exception then return 'null'::jsonb; end $$; diff --git a/proposals/stream-evidence-binding-2207/test_binding.py b/proposals/stream-evidence-binding-2207/test_binding.py index 6de3005..1675f29 100644 --- a/proposals/stream-evidence-binding-2207/test_binding.py +++ b/proposals/stream-evidence-binding-2207/test_binding.py @@ -70,7 +70,7 @@ def test_correction_and_provider_conflict_refuse_old_binding(self): self.setUp() self.sql(f"""INSERT INTO catalog_stream_runs(id,catalog_id,revision,scheduled_day,since,until) SELECT '{uid(13)}',catalog_id,revision,'2026-10-26','2026-10-24','2026-10-25' FROM catalog_stream_tracking; - INSERT INTO catalog_stream_observations SELECT '{uid(13)}',catalog_id,isrc,date,{streams},'{provider}','correction',retrieved_at+interval '1 hour' FROM catalog_stream_observations;""") + INSERT INTO catalog_stream_observations SELECT '{uid(13)}',catalog_id,isrc,date,{streams},'{provider}',repeat('b',64),retrieved_at+interval '1 hour' FROM catalog_stream_observations;""") self.assertEqual(self.call(), 'null') def test_tracking_disable_is_not_withdrawal(self): @@ -94,7 +94,7 @@ def test_source_and_membership_locks(self): 'DELETE FROM catalog_stream_runs', f"INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(11)}','{uid(14)}')", ] - self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content FROM context_source_versions") + self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content,retrieved_at) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content,retrieved_at FROM context_source_versions") self.hold_and_refuse(statements) def test_organization_revocation_lock(self): @@ -128,14 +128,14 @@ def test_current_result_and_extra_lineage_refused(self): self.sql('UPDATE context_documents SET revision=revision+1') self.assertEqual(self.call(), 'null') self.setUp() - self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content FROM context_source_versions; INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(11)}','{uid(14)}')") + self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content,retrieved_at) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content,retrieved_at FROM context_source_versions; INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(11)}','{uid(14)}')") self.assertEqual(self.call(), 'null') def test_corrected_version_is_explicit_and_old_selection_stays_stale(self): - self.sql(f"""UPDATE catalog_stream_observations SET streams=200,source_hash='correction'; - INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) + self.sql(f"""UPDATE catalog_stream_observations SET streams=200,source_hash=repeat('b',64); + INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content,retrieved_at) SELECT '{uid(14)}',owner_id,source_id,repeat('e',64),jsonb_set(content,'{{observations}}', - (SELECT jsonb_agg(to_jsonb(o) ORDER BY date) FROM catalog_stream_observations o)) FROM context_source_versions; + (SELECT jsonb_agg(to_jsonb(o) ORDER BY date) FROM catalog_stream_observations o)),retrieved_at FROM context_source_versions; INSERT INTO context_results(id,owner_id,attempt_id,subject_id,topic,reuse_key,status,evidence_kind,normalized_response,coverage) SELECT '{uid(15)}',r.owner_id,r.attempt_id,r.subject_id,r.topic,repeat('f',64),r.status,r.evidence_kind,v.content,r.coverage FROM context_results r CROSS JOIN context_source_versions v WHERE v.id='{uid(14)}'; @@ -146,7 +146,7 @@ def test_corrected_version_is_explicit_and_old_selection_stays_stale(self): self.assertEqual(value['evidence']['observations'][0]['streams'], 200) def test_link_insert_before_reader_lock_is_rechecked(self): - self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content FROM context_source_versions") + self.sql(f"INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content,retrieved_at) SELECT '{uid(14)}',owner_id,source_id,repeat('d',64),content,retrieved_at FROM context_source_versions") holder = subprocess.Popen(['psql', '-h', self.tmp.name, '-p', '55473', '-d', 'postgres', '-XAt', '-v', 'ON_ERROR_STOP=1'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -181,6 +181,154 @@ def test_active_replacement_cannot_bypass_withdrawn_scope(self): self.sql(f"INSERT INTO context_sources(id,owner_id,source_url,kind,withdrawn_at) SELECT '{uid(14)}',owner_id,source_url,kind,now() FROM context_sources") self.assertEqual(self.call(), 'null') + def refresh_fixture_bundle(self, since='2026-10-23', until='2026-10-25', version=9, result=11): + # Synthetic persistence fixture only; no production writer is exercised. + self.sql(f"""WITH rows AS ( + SELECT o.* FROM catalog_stream_observations o JOIN context_source_versions v + ON o.catalog_id=(v.content->>'catalogId')::uuid AND o.isrc=v.content->>'isrc' + WHERE v.id='{uid(version)}' AND date>='{since}' AND date<'{until}' + ), aggregate AS (SELECT jsonb_agg(to_jsonb(rows) ORDER BY date) AS observations, + min(retrieved_at) AS earliest,max(retrieved_at) AS latest FROM rows) + UPDATE context_source_versions v SET content=content || jsonb_build_object( + 'since','{since}','until','{until}','observations',a.observations, + 'providerRecordingId',(SELECT provider_recording_id FROM rows ORDER BY date LIMIT 1), + 'periods',jsonb_build_object('previous',jsonb_build_object('start','{since}', + 'end_exclusive','{since}'::date+('{until}'::date-'{since}'::date)/2), + 'current',jsonb_build_object('start','{since}'::date+('{until}'::date-'{since}'::date)/2, + 'end_exclusive','{until}'),'days',('{until}'::date-'{since}'::date)/2,'timezone','UTC'), + 'absentDates',(SELECT coalesce(jsonb_agg(d::date::text ORDER BY d),'[]'::jsonb) + FROM generate_series('{since}'::timestamp,'{until}'::timestamp-interval '1 day',interval '1 day') d + WHERE NOT EXISTS(SELECT 1 FROM rows WHERE date=d::date)), + 'nullDates',(SELECT coalesce(jsonb_agg(date::text ORDER BY date),'[]'::jsonb) FROM rows WHERE streams IS NULL), + 'retrievalRange',jsonb_build_object('earliest',a.earliest,'latest',a.latest)), retrieved_at=a.latest + FROM aggregate a WHERE v.id='{uid(version)}'; + UPDATE context_results r SET normalized_response=v.content FROM context_source_versions v + WHERE r.id='{uid(result)}' AND v.id='{uid(version)}';""") + + def new_fixture_result(self): + self.sql(f"""INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content,retrieved_at) + SELECT '{uid(14)}',owner_id,source_id,repeat('e',64),content,retrieved_at FROM context_source_versions WHERE id='{uid(9)}'; + INSERT INTO context_results(id,owner_id,attempt_id,subject_id,topic,reuse_key,status,evidence_kind,normalized_response,coverage) + SELECT '{uid(15)}',owner_id,attempt_id,subject_id,topic,repeat('f',64),status,evidence_kind,normalized_response,coverage + FROM context_results WHERE id='{uid(11)}'; + INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(15)}','{uid(14)}');""") + + def read_new(self): + return self.sql(f"SELECT stream_binding_proposal.read_binding('{uid(1)}','{uid(1)}','{uid(4)}','{uid(5)}','{uid(2)}','{uid(15)}','{uid(14)}',2)").stdout.strip() + + def test_multi_run_732_dates_and_over_limit(self): + self.sql(f"""DELETE FROM catalog_stream_observations; DELETE FROM catalog_stream_runs; + INSERT INTO catalog_stream_runs(catalog_id,revision,scheduled_day,since,until) + SELECT catalog_id,revision,'2024-01-01'::date+i*62,'2024-01-01'::date+i*62, + least('2024-01-01'::date+i*62+61,'2024-01-01'::date+731) + FROM catalog_stream_tracking CROSS JOIN generate_series(0,11) i; + INSERT INTO catalog_stream_observations + SELECT r.id,r.catalog_id,'USAAA2600001',d::date,1,'provider-a',repeat('a',64),'2026-10-25T00:00:00Z' + FROM catalog_stream_runs r CROSS JOIN LATERAL generate_series(r.since::timestamp,r.until::timestamp,interval '1 day') d;""") + self.refresh_fixture_bundle('2024-01-01','2026-01-02') + value = json.loads(self.call()) + self.assertEqual(len(value['evidence']['observations']), 732) + self.assertEqual(value['evidence']['periods']['days'], 366) + self.assertEqual(len({r['run_id'] for r in value['evidence']['observations']}), 12) + self.assertEqual(len(value['sourceVersionIds']), 1) + self.refresh_fixture_bundle('2024-01-01','2026-01-04') + self.assertEqual(self.call(), 'null') + + def test_window_replacement_stales_original_selection(self): + self.new_fixture_result() + self.refresh_fixture_bundle('2026-10-21','2026-10-25',14,15) + self.sql(f"SELECT accept_context_result('{uid(1)}',id,'{uid(15)}',1) FROM context_documents") + self.assertEqual(self.call(), 'null') + self.assertNotEqual(self.read_new(), 'null') + self.assertEqual(self.sql('SELECT count(*) FROM context_documents').stdout.strip(), '1') + + def test_all_dates_provider_switch_requires_disposition(self): + self.new_fixture_result() + self.sql("UPDATE catalog_stream_observations SET provider_recording_id='provider-b'") + self.refresh_fixture_bundle(version=14,result=15) + self.sql(f"SELECT accept_context_result('{uid(1)}',id,'{uid(15)}',1) FROM context_documents") + self.assertEqual(self.read_new(), 'null') + + def test_null_zero_missing_and_mixed_provider_rows(self): + for count in ['NULL', '0']: + self.setUp() + self.sql(f'UPDATE catalog_stream_observations SET streams={count}') + self.refresh_fixture_bundle() + value = json.loads(self.call())['evidence'] + self.assertEqual(value['absentDates'], ['2026-10-23']) + self.assertEqual(value['nullDates'], ['2026-10-24'] if count=='NULL' else []) + self.assertEqual(value['observations'][0]['streams'], None if count=='NULL' else 0) + self.sql("UPDATE catalog_stream_runs SET since='2026-10-23'; INSERT INTO catalog_stream_observations SELECT run_id,catalog_id,isrc,'2026-10-23',NULL,'provider-b',source_hash,retrieved_at FROM catalog_stream_observations") + self.refresh_fixture_bundle() + self.assertEqual(self.call(), 'null') + + def test_forged_run_range_and_malformed_retained_fields(self): + for sql in ["UPDATE catalog_stream_runs SET since='2026-10-25'", + "UPDATE catalog_stream_observations SET source_hash='not-a-hash'", + "UPDATE catalog_stream_observations SET provider_recording_id=' '", + "UPDATE catalog_stream_observations SET provider_recording_id=chr(9)||chr(10)", + "UPDATE catalog_stream_observations SET retrieved_at='infinity'"]: + self.setUp() + self.sql(sql) + self.refresh_fixture_bundle() + self.assertEqual(self.call(), 'null') + + def test_utf8_and_jsonb_byte_bound_refuses_without_truncation(self): + # Multi-byte provider text gives an otherwise structurally valid matching bundle. + self.sql("UPDATE catalog_stream_observations SET provider_recording_id=repeat('é',140000)") + self.refresh_fixture_bundle() + size = int(self.sql("SELECT octet_length(convert_to(content::text,'UTF8')) FROM context_source_versions").stdout.strip()) + self.assertGreater(size, 524288) + self.assertEqual(self.call(), 'null') + + def test_scope_shape_freshness_and_result_status(self): + mutations = ["UPDATE context_source_versions SET retrieved_at=now()", + "UPDATE context_source_versions SET content=jsonb_set(content,'{schemaVersion}',to_jsonb('1'::text))", + "UPDATE context_source_versions SET content=content||jsonb_build_object('inventedGrowth',999)", + "UPDATE context_results SET status='stale'", + "UPDATE context_results SET evidence_kind='customer_assertion'", + "UPDATE context_source_versions SET content=content||'{\"unexpected\":true}'::jsonb", + "UPDATE context_source_versions SET content=jsonb_set(content,'{metric}','\"spotify_annual_royalties\"')"] + for sql in mutations: + self.setUp() + self.sql(sql) + self.sql('UPDATE context_results SET normalized_response=(SELECT content FROM context_source_versions)') + self.assertEqual(self.call(), 'null') + + def test_cross_catalog_replacement_has_one_current_document(self): + self.sql(f"""INSERT INTO catalogs VALUES ('{uid(20)}','Second synthetic catalog',now()); + INSERT INTO account_catalogs VALUES ('{uid(1)}','{uid(20)}'); + INSERT INTO catalog_songs VALUES ('USAAA2600001','{uid(20)}'); + INSERT INTO context_resources(id,provider,resource_kind,provider_id,canonical_url) + VALUES ('{uid(21)}','recoup','catalog','{uid(20)}','urn:recoup:catalog:{uid(20)}'); + INSERT INTO context_subjects(id,kind,catalog_id) VALUES ('{uid(22)}','catalog','{uid(20)}'); + INSERT INTO context_requests(id,owner_id,created_by,resource_id,idempotency_key,input_fingerprint,input,status,output) + VALUES ('{uid(23)}','{uid(1)}','{uid(1)}','{uid(21)}','second',repeat('a',64),'{{}}','partial','{{"subjectIds":["{uid(22)}"]}}'); + INSERT INTO context_request_catalog_members(request_id,owner_id,catalog_subject_id,recording_subject_id,catalog_id,song_isrc) + VALUES ('{uid(23)}','{uid(1)}','{uid(22)}','{uid(5)}','{uid(20)}','USAAA2600001'); + INSERT INTO catalog_stream_tracking(catalog_id,owner_id) VALUES ('{uid(20)}','{uid(1)}'); + INSERT INTO catalog_stream_runs(id,catalog_id,revision,scheduled_day,since,until) + SELECT '{uid(24)}',catalog_id,revision,'2026-10-25','2026-10-24','2026-10-25' FROM catalog_stream_tracking WHERE catalog_id='{uid(20)}'; + INSERT INTO catalog_stream_observations SELECT '{uid(24)}','{uid(20)}',isrc,date,streams,provider_recording_id,source_hash,retrieved_at + FROM catalog_stream_observations WHERE catalog_id='{uid(2)}'; + INSERT INTO context_sources(id,owner_id,source_url,kind) + VALUES ('{uid(26)}','{uid(1)}','urn:recoup:retained-catalog-streams:{uid(20)}:{uid(5)}:all_dsps:worldwide','provider_metadata'); + INSERT INTO context_source_versions(id,owner_id,source_id,fingerprint,content,retrieved_at) + SELECT '{uid(25)}',owner_id,'{uid(26)}',repeat('e',64),jsonb_set(content,'{{catalogId}}','"{uid(20)}"'),retrieved_at + FROM context_source_versions WHERE id='{uid(9)}'; + INSERT INTO context_attempts(id,owner_id,request_id,module,attempt,status,provider,recipe_version,schema_version) + SELECT '{uid(28)}',owner_id,'{uid(23)}',module,attempt,status,provider,recipe_version,schema_version FROM context_attempts; + INSERT INTO context_results(id,owner_id,attempt_id,subject_id,topic,reuse_key,status,evidence_kind,normalized_response,coverage) + SELECT '{uid(27)}',owner_id,'{uid(28)}',subject_id,topic,repeat('f',64),status,evidence_kind,normalized_response,coverage FROM context_results; + INSERT INTO context_result_sources VALUES ('{uid(1)}','{uid(27)}','{uid(25)}');""") + self.refresh_fixture_bundle(version=25,result=27) + self.sql(f"SELECT accept_context_result('{uid(1)}',id,'{uid(27)}',1) FROM context_documents") + self.assertEqual(self.call(), 'null') + value = self.sql(f"SELECT stream_binding_proposal.read_binding('{uid(1)}','{uid(1)}','{uid(23)}','{uid(5)}','{uid(20)}','{uid(27)}','{uid(25)}',2)").stdout.strip() + self.assertNotEqual(value, 'null') + self.assertEqual(self.sql('SELECT count(*) FROM context_documents').stdout.strip(), '1') + self.assertEqual(self.sql('SELECT topic FROM context_documents').stdout.strip(), 'catalog_retained_streams') + def test_no_service_execute(self): self.assertEqual(self.sql("SELECT has_schema_privilege('service_role','stream_binding_proposal','USAGE')").stdout.strip(), 'f') From 6560ca76b048c4964335d2a8a282b34e59001bfc Mon Sep 17 00:00:00 2001 From: Sidney Swift <158200036+sidneyswift@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:09:09 -0400 Subject: [PATCH 5/5] Avoid persisting checkout credentials in proposal CI --- .github/workflows/stream-binding-proposal.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/stream-binding-proposal.yml b/.github/workflows/stream-binding-proposal.yml index 3619da2..c93429b 100644 --- a/.github/workflows/stream-binding-proposal.yml +++ b/.github/workflows/stream-binding-proposal.yml @@ -20,6 +20,8 @@ jobs: steps: - uses: actions/checkout@v4 timeout-minutes: 1 + with: + persist-credentials: false - name: Install PostgreSQL timeout-minutes: 12 run: |