fix(collaboration): bound legacy lease reads without losing visible continuations

.github/workflows/lease-list-bounds.yml

@@ -0,0 +1,73 @@
+name: Bounded collaboration lease reads
+on:
+  push:
+    branches: [fix/lease-list-bounds-20260929]
+  pull_request:
+    paths:
+      - '.github/workflows/lease-list-bounds.yml'
+      - 'src/msg/plugins/collaboration.py'
+      - 'tests/test_lease_list_bounds.py'
+      - 'tests/test_lease_list_deadline.py'
+  workflow_dispatch:
+permissions:
+  contents: read
+concurrency:
+  group: lease-list-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+jobs:
+  regression:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: msgtest
+          POSTGRES_PASSWORD: msgtest
+          POSTGRES_DB: postgres
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd "pg_isready -U msgtest -d postgres"
+          --health-interval 10s --health-timeout 5s --health-retries 5
+    env:
+      MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: Record exact source
+        run: |
+          mkdir -p artifacts
+          python3 - <<'PY'
+          import json, os, subprocess
+          from pathlib import Path
+          def git(*args):
+              return subprocess.check_output(['git', *args], text=True).strip()
+          Path('artifacts/source.json').write_text(json.dumps({
+              'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'),
+              'run_id': os.environ['GITHUB_RUN_ID'], 'attempt': os.environ['GITHUB_RUN_ATTEMPT']}, indent=2) + '\n')
+          PY
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.15'
+          allow-prereleases: true
+          cache: pip
+      - name: Install real dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo env NEEDRESTART_MODE=l apt-get install -y -qq age
+          python -m pip install -e '.[dev]'
+      - name: Require bounded reads and existing collaboration contracts
+        run: >-
+          python -m pytest -q
+          tests/test_lease_list_bounds.py tests/test_lease_list_deadline.py
+          tests/test_collaboration_current_visibility.py
+          tests/test_collaboration_surfaces.py tests/test_collaboration_resources.py
+          tests/test_collaboration_resource_surfaces.py
+          --junitxml=artifacts/lease-list-regressions.xml
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: lease-list-evidence
+          path: artifacts/
+          if-no-files-found: error

.github/workflows/read-page-regressions.yml

@@ -0,0 +1,85 @@
+name: Read page and recovery effect regressions
+on:
+  push:
+    branches: [fix/read-page-boundaries-20260929]
+  pull_request:
+    paths:
+      - '.github/workflows/read-page-regressions.yml'
+      - 'src/msg/plugins/discovery.py'
+      - 'src/msg/plugins/read_predicates.py'
+      - 'tests/test_read_page_boundaries.py'
+      - 'tests/test_read_entry_contract_matrix.py'
+      - 'tests/test_http_effect_order.py'
+  workflow_dispatch:
+permissions:
+  contents: read
+concurrency:
+  group: read-page-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+jobs:
+  regression:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: msgtest
+          POSTGRES_PASSWORD: msgtest
+          POSTGRES_DB: postgres
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd "pg_isready -U msgtest -d postgres"
+          --health-interval 10s --health-timeout 5s --health-retries 5
+    env:
+      MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: Record exact source identity
+        run: |
+          mkdir -p artifacts
+          python3 - <<'PY'
+          import json, os, subprocess
+          from pathlib import Path
+          def git(*args):
+              return subprocess.check_output(['git', *args], text=True).strip()
+          Path('artifacts/source.json').write_text(json.dumps({
+              'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'),
+              'run_id': os.environ['GITHUB_RUN_ID'], 'attempt': os.environ['GITHUB_RUN_ATTEMPT']}, indent=2) + '\n')
+          PY
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.15'
+          allow-prereleases: true
+          cache: pip
+      - name: Install real dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo env NEEDRESTART_MODE=l apt-get install -y -qq age
+          python -m pip install -e '.[dev]'
+      - name: Record the actual database collation and endpoint comparison
+        run: |
+          python - <<'PY'
+          import json, os, psycopg
+          from pathlib import Path
+          with psycopg.connect(os.environ['MSG_TEST_POSTGRES_URL_TEMPLATE'].format(database='postgres')) as db:
+              values = db.execute("SELECT version(), datcollate, %s::text < %s::text, %s::text COLLATE \"C\" < %s::text COLLATE \"C\" FROM pg_database WHERE datname=current_database()", ('r_ascii', '\uffff', '\U00010000', '\uffff')).fetchone()
+          assert values is not None
+          Path('artifacts/collation.json').write_text(json.dumps(dict(zip(
+              ('version', 'collation', 'ascii_below_old_sentinel', 'supplementary_below_old_sentinel_c'), values)), indent=2) + '\n')
+          PY
+      - name: Run real read entry and recovery regressions
+        run: >-
+          python -m pytest -q
+          tests/test_read_page_boundaries.py tests/test_read_entry_contract_matrix.py
+          tests/test_http_effect_order.py tests/test_recovery_runtime_generation.py
+          tests/test_legacy_routes_removed.py
+          --junitxml=artifacts/read-page-regressions.xml
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: read-page-evidence
+          path: artifacts/
+          if-no-files-found: error

docs/INTEGRATION_REGRESSION_PROGRESS.md

@@ -1,33 +1,82 @@
 # Integration regression closure — progress
 
-Refs #78, #80, #81, #82, #83, #85. Narrow follow-up to PR #176; combines PR #177
-and #178 without changing the active integration branch. No broad closure or
-production operation is claimed.
+Refs #78, #80, #81, #82, #83, #85. Integration entry: PR #176.
+Latest narrow follow-up: PR #180. Live verification, exact final head and merge
+state: #176 comment 5890775291 and #180. This note is a timestamped handoff,
+not a second completion authority or a claim of production acceptance.
 
-## Sources and completed work
+## 2026-09-29 13:05Z: current repair
 
-Integration base: `45470b5a46bccfc4f12ef27acccecc2f3bafce47`, tree
-`e75e7cf511a89d72bfdccf5ec39567824b569275`.
+Input integration head `de3cd872b71e062d6141916b11ea0bdfa28adca6`.
+Actual previous cloud checkout `9b727c16b893959e358811e543233df1608abb7b`,
+tree `4936af96eb805c4f90aad002008376425a2effa4`.
 
-The merge commit retains both independent histories:
-- #177 head `769c124c83a01a648d3d30ef5d4a19fda98bedb6`: bounded transfer status
-  cursors. Focused cloud run `36561282522`: 24/24, no failures/errors/skips.
-- #178 head `517f7e296fa7ef504d2066980454654ed0634598`: positive integer client
-  chunk-size checks. RED run `36561926956`: 14 intended failures, 9 passes;
-  final focused run pending. No protocol/journal/schema changes.
+Independently downloaded all four artifacts of run **36564681470**, checked
+provider SHA256, each JUnit msg.nodeid against selected/all, common collection,
+exact disjoint union and no duplicates: **2407 unique cases, 6 failures,
+0 errors, 0 skipped**. Four effect-order failures and two signed descending
+read_query@1/@3 failures; the earlier webhook-overlap failures are absent.
 
-## Base full-suite evidence
+| Shard | Artifact | ZIP SHA256 |
+| --- | --- | --- |
+| 0 | 11031896647 | fd8c2110ab4240246af4d3b323b5676b7458af5016b6ed18c940010772a691e0 |
+| 1 | 11031762212 | 95bf9181ea1c39acd3d9ea62edc0d194104e116d93a699b38c01a53d704f0dbb |
+| 2 | 11031389353 | 7a00d14bea7a34c8d1439f936e8d78813c9d76f973fec0e3e518e5ffd4c52a1d |
+| 3 | 11031559533 | d7c9c081408ad09b01be264e3e3ac6df4d84f020a4bab4d9f2d2f3eca5ddc5df |
+
+### Submitted implementation
+
+Source commit `12e829a6a7b90c279e9b15cd74cece63978ca387`, tree
+`07449e7ae96399ebcbcd1ae33a53de3d9ab5d8f5`, in #180 based on #176:
+
+- Remove artificial initial empty-string/U+FFFF keyset endpoints. No seek on
+  the first page; later scans/cursors keep actual SQL keys. Ordering, principal,
+  query/snapshot binding, authorization and budgets are unchanged.
+- Retain the real runtime-generation SQL read in the effect test. Precisely
+  forbid every other SQL query and executor call, check each request actually
+  reads generation, and also test stale generations fail closed. Recovery
+  middleware is not weakened, bypassed or mocked to succeed.
+- Add list/search/read_query@1/@2/@3 checks for all three sort keys in both
+  directions, Unicode names, exact database order, nonoverlapping continuation
+  and empty results. Existing signed four-transport tests remain unchanged.
+
+Source-only preparation run 36571871534 published just one reviewed source
+blob. Artifact11035120921 ZIP SHA256
+`02f5421eb68c126198dd1fe0f6936918f339470d9350fa201846a5ed0434d47d` was checked
+independently, including byte-for-byte source equality. Source blob
+`985c0c222220bfc23b0e28b4e54871bef5e7885d`; SHA256
+`2cc2b605e6590d61e0f65e63f613f75e1488b2d40c77224d1b77b4012694ffc0`.
+The auxiliary workflow neither ran project code nor updated a branch and is
+absent from the product tree.
 
-Run `36559785404` executed 2259 unique test node IDs in four disjoint partitions.
-All four manifests and JUnit node IDs were independently compared: exact union,
-no missing or duplicate cases. Four failures, no errors or skipped test cases:
+### Verification in progress; no GREEN claim yet
 
-- `test_legacy_routes_removed::test_public_views_require_read_effects`.
-- `test_webhook_overlap::test_overlap_one_job_per_event_recipient_channel_and_pinned_revocation[endpoint]`.
-- The same webhook case with `[unsubscribe]`.
-- `test_webhook_overlap::test_different_recipients_and_channels_remain_distinct`.
+Initial full run 36572191345 and integration run 36572191005 were triggered.
+Focused run 36572191224 failed in the newly added collation diagnostic before
+pytest: lc_collate is not a supported current_setting parameter. The follow-up
+reads pg_database.datcollate for current_database(), as documented in the
+PostgreSQL 16 system catalog, and fixes the health probe's database name.
+Runtime source and test assertions are unchanged by that CI correction.
+New runs must identify their actual final head/tree; older partial or successful
+runs do not satisfy the final combined gate.
 
-Artifacts were independently downloaded and SHA256-verified:
+## Preserved earlier history
+
+Earlier integration base `45470b5a46bccfc4f12ef27acccecc2f3bafce47`, tree
+`e75e7cf511a89d72bfdccf5ec39567824b569275`. PR #179 retained both histories
+and was merged into #176 as de3cd872; no duplicate implementations are needed:
+
+- #177 head `769c124c83a01a648d3d30ef5d4a19fda98bedb6`: bounded transfer
+  status cursors. Historical focused run 36561282522: 24/24, zero failures,
+  errors or skips. See its original evidence note for full provenance.
+- #178 head `517f7e296fa7ef504d2066980454654ed0634598`: positive integer
+  client chunk sizes. Historical RED36561926956: 14 intended failures,
+  9 passes. Original transfer/client evidence remains authoritative for those
+  narrow changes, not current integrated acceptance.
+
+Earlier full run36559785404 had 2259 unique cases, four failures, no errors or
+skips: one read-effect-before-lookup case and three webhook-overlap cases.
+Exact manifests/JUnit union and provider hashes were independently checked:
 
 | Shard | Artifact | ZIP SHA256 |
 | --- | --- | --- |
@@ -36,29 +85,19 @@ Artifacts were independently downloaded and SHA256-verified:
 | 2 | 11029802392 | 18fbb03c4531e3128ffe1e0de45602aede0532132406c1b96388ed5c934b7df2 |
 | 3 | 11030251180 | e2d4990543cbf937af745f0b9429c260c024a64dee51065561a652e2171c7370 |
 
-The full gate therefore fails; the later conformance/build stages are not
-claimed complete. The same read-route failure exists on #177's shard2 run
-`36561282628` (577 cases, one failure), independent of its green focused tests.
-
-## Remaining corrections
-
-1. `http_routes.py` resolves ordinary resource paths before checking the
-   selected discovery operation's read effect. A missing path returns 404
-   instead of rejecting changed effects with 405. Move the shared effect check
-   before lookups and retain current request/resource authorization. New tests
-   cover GET/HEAD, transaction/external effects and metadata/executor guards.
-2. `communication.py::_domain_delivery_exists` uses a textual punctuation range
-   for prefix matching. The real PostgreSQL overlap tests hit jobs_dedupe_key
-   conflicts. Investigate collation-safe literal prefix matching while retaining
-   historical scope-key recognition, exact current authorization and pinned
-   endpoint/subscription generations. Never turn unique conflicts into a blanket
-   swallowed error or weaken the unique constraint.
-
-The focused workflow preserves the exact public checkout as an artifact before
-running tests, so failures can be reviewed without relying on an older sdist.
-It has contents:read only and never pushes commits or touches production data.
-
-Status: combined histories, regression-first checks and workflow prepared.
-Runtime read-route and webhook corrections, exact final-head focused verification,
-full four-shard gate, conformance and build are pending. All project execution is
-in cloud CI; local work only edits text and inspects/hashes artifacts.
+That historical run was RED, with later conformance/build not claimed complete.
+#179 moved read-effect checks before resource lookups and corrected literal
+webhook prefix matching without removing unique constraints or swallowing
+conflicts. The current combined six failures above supersede that old triage.
+
+## Unchanged completion and execution boundaries
+
+Require final focused suites, all four shards, exact disjoint node-ID coverage,
+conformance/actual tokenizer, wheel/sdist checks and applicable specialist gates.
+No removed tests, skips, relaxed authentication or substituted old-head greens.
+All project execution happens in cloud CI; local work only edits source text
+and independently reads/hashes artifacts.
+
+#78/#80/#81/#82/#83/#85 broad acceptance and #64/#65/#68–70/#84 field evidence
+remain open. No production deployment, Root/PIN access, money movement, real
+outbound delivery, backup/key destruction or recovery promotion is performed.

src/msg/plugins/collaboration.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 from datetime import timedelta
+import time
 
 from msg.core.codec import canonical, loads, parse_time, wire
 from msg.core.errors import Failure, require
@@ -203,27 +204,37 @@ async def lease_get(ctx, request, tx):
     async def lease_list(ctx, request, tx):
         holder = await _self(app, ctx, request, tx)
         limit = request.arguments.get('limit', 50)
-        rows = tx.rows('SELECT id,target,status,expires_at,body FROM collaboration_leases '
-                       'WHERE holder=? AND id>? ORDER BY id',
-                       (holder, request.arguments.get('after', '')))
+        position = request.arguments.get('after', '')
         records = []
         more = False
-        for _, target, status, expires, raw in rows:
-            try:
-                await _safe_ref(app, ctx, request, tx, target)
-            except Failure as exc:
-                if exc.code in {'permission_denied', 'credential_ceiling', 'certificate_gate',
-                                'delegation_scope', 'ancestor_inactive', 'not_found',
-                                'collaboration_ref_forbidden'}:
-                    continue
-                raise
-            record = loads(raw)
-            record['effective_status'] = ('expired' if status == 'active' and
-                                          parse_time(expires) <= ctx.now else status)
-            if len(records) == limit:
-                more = True
+        scanned = 0
+        while not more:
+            require(time.monotonic() < ctx.deadline_monotonic, 'query_cost_exceeded')
+            rows = tx.rows('SELECT id,target,status,expires_at,body FROM collaboration_leases '
+                           'WHERE holder=? AND id>? ORDER BY id LIMIT 128',
+                           (holder, position))
+            for lease_id, target, status, expires, raw in rows:
+                scanned += 1
+                require(scanned <= 4096 and time.monotonic() < ctx.deadline_monotonic,
+                        'query_cost_exceeded')
+                position = lease_id
+                try:
+                    await _safe_ref(app, ctx, request, tx, target)
+                except Failure as exc:
+                    if exc.code in {'permission_denied', 'credential_ceiling', 'certificate_gate',
+                                    'delegation_scope', 'ancestor_inactive', 'not_found',
+                                    'collaboration_ref_forbidden'}:
+                        continue
+                    raise
+                if len(records) == limit:
+                    more = True
+                    break
+                record = loads(raw)
+                record['effective_status'] = ('expired' if status == 'active' and
+                                              parse_time(expires) <= ctx.now else status)
+                records.append(record)
+            if len(rows) < 128:
                 break
-            records.append(record)
         return HandlerOutput(data={'items': records,
                                    'next_after': records[-1]['id'] if more else None})
 

src/msg/plugins/discovery.py

@@ -505,7 +505,7 @@ async def list_items(ctx,request,tx, *, arguments=None,budget=None):
             position,snapshot=app.cursors.decode_page(a['cursor'],request.operation,
                                                        query_args,principal,ctx.now)
         else:
-            position=app.cursors.decode(a['cursor'],'page',query_hash) if a.get('cursor') else (['\uffff','\uffff'] if descending else ['', ''])
+            position=app.cursors.decode(a['cursor'],'page',query_hash) if a.get('cursor') else None
             snapshot=ctx.now
         from msg.plugins.read_predicates import read_predicates
         filters,parameters=await read_predicates(app,tx,a,parent)
@@ -517,8 +517,11 @@ async def list_items(ctx,request,tx, *, arguments=None,budget=None):
         scanned=0
         while len(values)<=limit:
             require(time.monotonic()<ctx.deadline_monotonic,'query_cost_exceeded')
-            sql=f"SELECT r.body,{column},r.id FROM resources r WHERE {' AND '.join(filters)} AND ({column},r.id){comparison}(?,?) ORDER BY {column} {ordering},r.id {ordering} LIMIT 128"
-            rows=tx.rows(sql,(*parameters,*last_position))
+            # No text value is a universal endpoint under database collations.
+            # Seek only from an actual cursor or the last scanned resource.
+            seek=f" AND ({column},r.id){comparison}(?,?)" if last_position is not None else ''
+            sql=f"SELECT r.body,{column},r.id FROM resources r WHERE {' AND '.join(filters)}{seek} ORDER BY {column} {ordering},r.id {ordering} LIMIT 128"
+            rows=tx.rows(sql,(*parameters,*(last_position if last_position is not None else ())))
             if not rows:
                 break
             for raw,order,rid in rows: