diff --git a/.github/workflows/lease-list-bounds.yml b/.github/workflows/lease-list-bounds.yml new file mode 100644 index 00000000..f9bbac6e --- /dev/null +++ b/.github/workflows/lease-list-bounds.yml @@ -0,0 +1,73 @@ +name: Bounded collaboration lease reads +on: + push: + branches: [fix/lease-list-bounds-20260929] + pull_request: + paths: + - '.github/workflows/lease-list-bounds.yml' + - 'src/msg/plugins/collaboration.py' + - 'tests/test_lease_list_bounds.py' + - 'tests/test_lease_list_deadline.py' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: lease-list-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + regression: + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest -d postgres" + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Record exact source + run: | + mkdir -p artifacts + python3 - <<'PY' + import json, os, subprocess + from pathlib import Path + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'), + 'run_id': os.environ['GITHUB_RUN_ID'], 'attempt': os.environ['GITHUB_RUN_ATTEMPT']}, indent=2) + '\n') + PY + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install real dependencies + run: | + sudo apt-get update -qq + sudo env NEEDRESTART_MODE=l apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Require bounded reads and existing collaboration contracts + run: >- + python -m pytest -q + tests/test_lease_list_bounds.py tests/test_lease_list_deadline.py + tests/test_collaboration_current_visibility.py + tests/test_collaboration_surfaces.py tests/test_collaboration_resources.py + tests/test_collaboration_resource_surfaces.py + --junitxml=artifacts/lease-list-regressions.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lease-list-evidence + path: artifacts/ + if-no-files-found: error diff --git a/.github/workflows/read-page-regressions.yml b/.github/workflows/read-page-regressions.yml new file mode 100644 index 00000000..a0a1c30b --- /dev/null +++ b/.github/workflows/read-page-regressions.yml @@ -0,0 +1,85 @@ +name: Read page and recovery effect regressions +on: + push: + branches: [fix/read-page-boundaries-20260929] + pull_request: + paths: + - '.github/workflows/read-page-regressions.yml' + - 'src/msg/plugins/discovery.py' + - 'src/msg/plugins/read_predicates.py' + - 'tests/test_read_page_boundaries.py' + - 'tests/test_read_entry_contract_matrix.py' + - 'tests/test_http_effect_order.py' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: read-page-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + regression: + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest -d postgres" + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Record exact source identity + run: | + mkdir -p artifacts + python3 - <<'PY' + import json, os, subprocess + from pathlib import Path + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'), + 'run_id': os.environ['GITHUB_RUN_ID'], 'attempt': os.environ['GITHUB_RUN_ATTEMPT']}, indent=2) + '\n') + PY + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install real dependencies + run: | + sudo apt-get update -qq + sudo env NEEDRESTART_MODE=l apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Record the actual database collation and endpoint comparison + run: | + python - <<'PY' + import json, os, psycopg + from pathlib import Path + with psycopg.connect(os.environ['MSG_TEST_POSTGRES_URL_TEMPLATE'].format(database='postgres')) as db: + values = db.execute("SELECT version(), datcollate, %s::text < %s::text, %s::text COLLATE \"C\" < %s::text COLLATE \"C\" FROM pg_database WHERE datname=current_database()", ('r_ascii', '\uffff', '\U00010000', '\uffff')).fetchone() + assert values is not None + Path('artifacts/collation.json').write_text(json.dumps(dict(zip( + ('version', 'collation', 'ascii_below_old_sentinel', 'supplementary_below_old_sentinel_c'), values)), indent=2) + '\n') + PY + - name: Run real read entry and recovery regressions + run: >- + python -m pytest -q + tests/test_read_page_boundaries.py tests/test_read_entry_contract_matrix.py + tests/test_http_effect_order.py tests/test_recovery_runtime_generation.py + tests/test_legacy_routes_removed.py + --junitxml=artifacts/read-page-regressions.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: read-page-evidence + path: artifacts/ + if-no-files-found: error diff --git a/docs/INTEGRATION_REGRESSION_PROGRESS.md b/docs/INTEGRATION_REGRESSION_PROGRESS.md index 071f3fc5..2327f4d2 100644 --- a/docs/INTEGRATION_REGRESSION_PROGRESS.md +++ b/docs/INTEGRATION_REGRESSION_PROGRESS.md @@ -1,33 +1,82 @@ # Integration regression closure — progress -Refs #78, #80, #81, #82, #83, #85. Narrow follow-up to PR #176; combines PR #177 -and #178 without changing the active integration branch. No broad closure or -production operation is claimed. +Refs #78, #80, #81, #82, #83, #85. Integration entry: PR #176. +Latest narrow follow-up: PR #180. Live verification, exact final head and merge +state: #176 comment 5890775291 and #180. This note is a timestamped handoff, +not a second completion authority or a claim of production acceptance. -## Sources and completed work +## 2026-09-29 13:05Z: current repair -Integration base: `45470b5a46bccfc4f12ef27acccecc2f3bafce47`, tree -`e75e7cf511a89d72bfdccf5ec39567824b569275`. +Input integration head `de3cd872b71e062d6141916b11ea0bdfa28adca6`. +Actual previous cloud checkout `9b727c16b893959e358811e543233df1608abb7b`, +tree `4936af96eb805c4f90aad002008376425a2effa4`. -The merge commit retains both independent histories: -- #177 head `769c124c83a01a648d3d30ef5d4a19fda98bedb6`: bounded transfer status - cursors. Focused cloud run `36561282522`: 24/24, no failures/errors/skips. -- #178 head `517f7e296fa7ef504d2066980454654ed0634598`: positive integer client - chunk-size checks. RED run `36561926956`: 14 intended failures, 9 passes; - final focused run pending. No protocol/journal/schema changes. +Independently downloaded all four artifacts of run **36564681470**, checked +provider SHA256, each JUnit msg.nodeid against selected/all, common collection, +exact disjoint union and no duplicates: **2407 unique cases, 6 failures, +0 errors, 0 skipped**. Four effect-order failures and two signed descending +read_query@1/@3 failures; the earlier webhook-overlap failures are absent. -## Base full-suite evidence +| Shard | Artifact | ZIP SHA256 | +| --- | --- | --- | +| 0 | 11031896647 | fd8c2110ab4240246af4d3b323b5676b7458af5016b6ed18c940010772a691e0 | +| 1 | 11031762212 | 95bf9181ea1c39acd3d9ea62edc0d194104e116d93a699b38c01a53d704f0dbb | +| 2 | 11031389353 | 7a00d14bea7a34c8d1439f936e8d78813c9d76f973fec0e3e518e5ffd4c52a1d | +| 3 | 11031559533 | d7c9c081408ad09b01be264e3e3ac6df4d84f020a4bab4d9f2d2f3eca5ddc5df | + +### Submitted implementation + +Source commit `12e829a6a7b90c279e9b15cd74cece63978ca387`, tree +`07449e7ae96399ebcbcd1ae33a53de3d9ab5d8f5`, in #180 based on #176: + +- Remove artificial initial empty-string/U+FFFF keyset endpoints. No seek on + the first page; later scans/cursors keep actual SQL keys. Ordering, principal, + query/snapshot binding, authorization and budgets are unchanged. +- Retain the real runtime-generation SQL read in the effect test. Precisely + forbid every other SQL query and executor call, check each request actually + reads generation, and also test stale generations fail closed. Recovery + middleware is not weakened, bypassed or mocked to succeed. +- Add list/search/read_query@1/@2/@3 checks for all three sort keys in both + directions, Unicode names, exact database order, nonoverlapping continuation + and empty results. Existing signed four-transport tests remain unchanged. + +Source-only preparation run 36571871534 published just one reviewed source +blob. Artifact11035120921 ZIP SHA256 +`02f5421eb68c126198dd1fe0f6936918f339470d9350fa201846a5ed0434d47d` was checked +independently, including byte-for-byte source equality. Source blob +`985c0c222220bfc23b0e28b4e54871bef5e7885d`; SHA256 +`2cc2b605e6590d61e0f65e63f613f75e1488b2d40c77224d1b77b4012694ffc0`. +The auxiliary workflow neither ran project code nor updated a branch and is +absent from the product tree. -Run `36559785404` executed 2259 unique test node IDs in four disjoint partitions. -All four manifests and JUnit node IDs were independently compared: exact union, -no missing or duplicate cases. Four failures, no errors or skipped test cases: +### Verification in progress; no GREEN claim yet -- `test_legacy_routes_removed::test_public_views_require_read_effects`. -- `test_webhook_overlap::test_overlap_one_job_per_event_recipient_channel_and_pinned_revocation[endpoint]`. -- The same webhook case with `[unsubscribe]`. -- `test_webhook_overlap::test_different_recipients_and_channels_remain_distinct`. +Initial full run 36572191345 and integration run 36572191005 were triggered. +Focused run 36572191224 failed in the newly added collation diagnostic before +pytest: lc_collate is not a supported current_setting parameter. The follow-up +reads pg_database.datcollate for current_database(), as documented in the +PostgreSQL 16 system catalog, and fixes the health probe's database name. +Runtime source and test assertions are unchanged by that CI correction. +New runs must identify their actual final head/tree; older partial or successful +runs do not satisfy the final combined gate. -Artifacts were independently downloaded and SHA256-verified: +## Preserved earlier history + +Earlier integration base `45470b5a46bccfc4f12ef27acccecc2f3bafce47`, tree +`e75e7cf511a89d72bfdccf5ec39567824b569275`. PR #179 retained both histories +and was merged into #176 as de3cd872; no duplicate implementations are needed: + +- #177 head `769c124c83a01a648d3d30ef5d4a19fda98bedb6`: bounded transfer + status cursors. Historical focused run 36561282522: 24/24, zero failures, + errors or skips. See its original evidence note for full provenance. +- #178 head `517f7e296fa7ef504d2066980454654ed0634598`: positive integer + client chunk sizes. Historical RED36561926956: 14 intended failures, + 9 passes. Original transfer/client evidence remains authoritative for those + narrow changes, not current integrated acceptance. + +Earlier full run36559785404 had 2259 unique cases, four failures, no errors or +skips: one read-effect-before-lookup case and three webhook-overlap cases. +Exact manifests/JUnit union and provider hashes were independently checked: | Shard | Artifact | ZIP SHA256 | | --- | --- | --- | @@ -36,29 +85,19 @@ Artifacts were independently downloaded and SHA256-verified: | 2 | 11029802392 | 18fbb03c4531e3128ffe1e0de45602aede0532132406c1b96388ed5c934b7df2 | | 3 | 11030251180 | e2d4990543cbf937af745f0b9429c260c024a64dee51065561a652e2171c7370 | -The full gate therefore fails; the later conformance/build stages are not -claimed complete. The same read-route failure exists on #177's shard2 run -`36561282628` (577 cases, one failure), independent of its green focused tests. - -## Remaining corrections - -1. `http_routes.py` resolves ordinary resource paths before checking the - selected discovery operation's read effect. A missing path returns 404 - instead of rejecting changed effects with 405. Move the shared effect check - before lookups and retain current request/resource authorization. New tests - cover GET/HEAD, transaction/external effects and metadata/executor guards. -2. `communication.py::_domain_delivery_exists` uses a textual punctuation range - for prefix matching. The real PostgreSQL overlap tests hit jobs_dedupe_key - conflicts. Investigate collation-safe literal prefix matching while retaining - historical scope-key recognition, exact current authorization and pinned - endpoint/subscription generations. Never turn unique conflicts into a blanket - swallowed error or weaken the unique constraint. - -The focused workflow preserves the exact public checkout as an artifact before -running tests, so failures can be reviewed without relying on an older sdist. -It has contents:read only and never pushes commits or touches production data. - -Status: combined histories, regression-first checks and workflow prepared. -Runtime read-route and webhook corrections, exact final-head focused verification, -full four-shard gate, conformance and build are pending. All project execution is -in cloud CI; local work only edits text and inspects/hashes artifacts. +That historical run was RED, with later conformance/build not claimed complete. +#179 moved read-effect checks before resource lookups and corrected literal +webhook prefix matching without removing unique constraints or swallowing +conflicts. The current combined six failures above supersede that old triage. + +## Unchanged completion and execution boundaries + +Require final focused suites, all four shards, exact disjoint node-ID coverage, +conformance/actual tokenizer, wheel/sdist checks and applicable specialist gates. +No removed tests, skips, relaxed authentication or substituted old-head greens. +All project execution happens in cloud CI; local work only edits source text +and independently reads/hashes artifacts. + +#78/#80/#81/#82/#83/#85 broad acceptance and #64/#65/#68–70/#84 field evidence +remain open. No production deployment, Root/PIN access, money movement, real +outbound delivery, backup/key destruction or recovery promotion is performed. diff --git a/src/msg/plugins/collaboration.py b/src/msg/plugins/collaboration.py index 90c20cc8..f978dc61 100644 --- a/src/msg/plugins/collaboration.py +++ b/src/msg/plugins/collaboration.py @@ -2,6 +2,7 @@ from __future__ import annotations from datetime import timedelta +import time from msg.core.codec import canonical, loads, parse_time, wire from msg.core.errors import Failure, require @@ -203,27 +204,37 @@ async def lease_get(ctx, request, tx): async def lease_list(ctx, request, tx): holder = await _self(app, ctx, request, tx) limit = request.arguments.get('limit', 50) - rows = tx.rows('SELECT id,target,status,expires_at,body FROM collaboration_leases ' - 'WHERE holder=? AND id>? ORDER BY id', - (holder, request.arguments.get('after', ''))) + position = request.arguments.get('after', '') records = [] more = False - for _, target, status, expires, raw in rows: - try: - await _safe_ref(app, ctx, request, tx, target) - except Failure as exc: - if exc.code in {'permission_denied', 'credential_ceiling', 'certificate_gate', - 'delegation_scope', 'ancestor_inactive', 'not_found', - 'collaboration_ref_forbidden'}: - continue - raise - record = loads(raw) - record['effective_status'] = ('expired' if status == 'active' and - parse_time(expires) <= ctx.now else status) - if len(records) == limit: - more = True + scanned = 0 + while not more: + require(time.monotonic() < ctx.deadline_monotonic, 'query_cost_exceeded') + rows = tx.rows('SELECT id,target,status,expires_at,body FROM collaboration_leases ' + 'WHERE holder=? AND id>? ORDER BY id LIMIT 128', + (holder, position)) + for lease_id, target, status, expires, raw in rows: + scanned += 1 + require(scanned <= 4096 and time.monotonic() < ctx.deadline_monotonic, + 'query_cost_exceeded') + position = lease_id + try: + await _safe_ref(app, ctx, request, tx, target) + except Failure as exc: + if exc.code in {'permission_denied', 'credential_ceiling', 'certificate_gate', + 'delegation_scope', 'ancestor_inactive', 'not_found', + 'collaboration_ref_forbidden'}: + continue + raise + if len(records) == limit: + more = True + break + record = loads(raw) + record['effective_status'] = ('expired' if status == 'active' and + parse_time(expires) <= ctx.now else status) + records.append(record) + if len(rows) < 128: break - records.append(record) return HandlerOutput(data={'items': records, 'next_after': records[-1]['id'] if more else None}) diff --git a/src/msg/plugins/discovery.py b/src/msg/plugins/discovery.py index 56c155c7..985c0c22 100644 --- a/src/msg/plugins/discovery.py +++ b/src/msg/plugins/discovery.py @@ -505,7 +505,7 @@ async def list_items(ctx,request,tx, *, arguments=None,budget=None): position,snapshot=app.cursors.decode_page(a['cursor'],request.operation, query_args,principal,ctx.now) else: - position=app.cursors.decode(a['cursor'],'page',query_hash) if a.get('cursor') else (['\uffff','\uffff'] if descending else ['', '']) + position=app.cursors.decode(a['cursor'],'page',query_hash) if a.get('cursor') else None snapshot=ctx.now from msg.plugins.read_predicates import read_predicates filters,parameters=await read_predicates(app,tx,a,parent) @@ -517,8 +517,11 @@ async def list_items(ctx,request,tx, *, arguments=None,budget=None): scanned=0 while len(values)<=limit: require(time.monotonic()