From 04299d56c96323a42531665fa8505bca38840105 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 12:03:38 +0200 Subject: [PATCH 01/32] fix(storage): harden durable change-train admission Problem: released historical train manifests admitted a later durable tier using only SQLite integrity, so a healthy database from another archive could pass. Lifecycle probes also materialized every failed raw before applying their sample limit. What changed: require immutable archive identity and the train's frozen historical fresh-schema proof for forward admission, record the historical and current targets in the execution receipt, and reuse one captured evidence object per tier while reconciling historical manifests. Aggregate lifecycle counts in SQLite and limit detailed sample selection in SQL. Compatibility/migration: current-target continuity remains exact; later versions are admitted only for the same archive with retained historical schema evidence. Ref polylogue-dcrmm Co-Authored-By: Codex --- polylogue/storage/raw_failure_lifecycle.py | 58 ++++++--- .../storage/sqlite/durable_change_train.py | 117 ++++++++++++++++-- tests/unit/daemon/test_raw_failure_sample.py | 39 ++++++ .../unit/storage/test_durable_change_train.py | 47 +++++++ 4 files changed, 233 insertions(+), 28 deletions(-) diff --git a/polylogue/storage/raw_failure_lifecycle.py b/polylogue/storage/raw_failure_lifecycle.py index 7a509e3d00..fe1d198c68 100644 --- a/polylogue/storage/raw_failure_lifecycle.py +++ b/polylogue/storage/raw_failure_lifecycle.py @@ -165,7 +165,23 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra OR r.validation_status = 'failed' ORDER BY r.acquired_at_ms DESC, r.raw_id DESC """ - failed_rows = conn.execute(failure_query).fetchall() + summary_rows = conn.execute( + f""" + WITH failed AS ({failure_query}) + SELECT origin, validation_status, artifact_kind, support_status, COUNT(*) + FROM failed + GROUP BY origin, validation_status, artifact_kind, support_status + """ + ).fetchall() + sample_rows = conn.execute( + f""" + WITH failed AS ({failure_query}) + SELECT raw_id, origin, validation_status, artifact_kind, support_status + FROM failed + LIMIT ? + """, + (max(0, sample_limit),), + ).fetchall() except sqlite3.Error as exc: logger.warning("could not read raw failure lifecycle", exc_info=exc) return RawFailureLifecycleSnapshot(False, reason=f"could not read raw failure lifecycle: {exc}") @@ -176,25 +192,33 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra by_artifact_kind: Counter[str] = Counter() counts: Counter[str] = Counter() samples: list[dict[str, str | None]] = [] - for row in failed_rows: + for row in summary_rows: + origin = str(row[0] or "unknown") + artifact_kind = str(row[2]) if row[2] is not None else None + support_status = str(row[3]) if row[3] is not None else None + validation_failed = str(row[1] or "") == "failed" + lifecycle = _lifecycle(artifact_kind, support_status, validation_failed=validation_failed) + count = int(row[4]) + counts[lifecycle] += count + by_origin[origin] += count + by_artifact_kind[artifact_kind or ""] += count + for row in sample_rows: origin = str(row[1] or "unknown") artifact_kind = str(row[3]) if row[3] is not None else None support_status = str(row[4]) if row[4] is not None else None - validation_failed = str(row[2] or "") == "failed" - lifecycle = _lifecycle(artifact_kind, support_status, validation_failed=validation_failed) - counts[lifecycle] += 1 - by_origin[origin] += 1 - by_artifact_kind[artifact_kind or ""] += 1 - if len(samples) < max(0, sample_limit): - samples.append( - { - "raw_id": str(row[0]), - "origin": origin, - "artifact_kind": artifact_kind, - "support_status": support_status, - "lifecycle": lifecycle, - } - ) + samples.append( + { + "raw_id": str(row[0]), + "origin": origin, + "artifact_kind": artifact_kind, + "support_status": support_status, + "lifecycle": _lifecycle( + artifact_kind, + support_status, + validation_failed=str(row[2] or "") == "failed", + ), + } + ) return RawFailureLifecycleSnapshot( available=True, parse_failures=parse_failures, diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 2816f3bf2c..9e1135956c 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -99,6 +99,20 @@ class DurableChangeTrainExecution: train: DurableChangeTrain | None manifest_path: Path | None migration_result: MigrationResult | None + forward_version_receipt: DurableForwardVersionReceipt | None = None + + +@dataclass(frozen=True, slots=True) +class DurableForwardVersionReceipt: + """Evidence that a historical released train admits a later live tier.""" + + tier: ArchiveTier + historical_train_id: str + historical_target_version: int + current_target_version: int + observed_live_version: int + historical_schema_inventory_sha256: str + archive_identity_digest: str def durable_migration_sidecar_name(slot: int) -> str: @@ -1362,11 +1376,16 @@ def _open_existing_tier(tier_path: Path) -> sqlite3.Connection: raise DurableChangeTrainError("durable tier could not be opened without initialization") from exc -def _verify_persisted_live_tier_continuity(conn: sqlite3.Connection, train: DurableChangeTrain) -> None: +def _verify_persisted_live_tier_continuity( + conn: sqlite3.Connection, + train: DurableChangeTrain, + *, + actual: DurableDatabaseEvidence | None = None, +) -> None: """Prove the exact reopened connection still names the persisted durable tier.""" if train.apply_evidence is None: raise DurableChangeTrainError(f"{train.state.value} train lacks post-apply continuity evidence") - actual = capture_durable_database_evidence(conn, train.tier) + actual = actual or capture_durable_database_evidence(conn, train.tier) expected = train.apply_evidence.post if actual.user_version != train.target_version: raise DurableChangeTrainError( @@ -1380,11 +1399,41 @@ def _verify_persisted_live_tier_continuity(conn: sqlite3.Connection, train: Dura ) from exc -def _verify_released_train_live_tier(archive_root: Path, conn: sqlite3.Connection, train: DurableChangeTrain) -> None: +def _historical_schema_evidence(train: DurableChangeTrain) -> DurableFreshDDLParityProof: + """Return the immutable historical schema proof a forward admission needs.""" + if train.apply_evidence is None or train.proof is None or train.fresh_ddl_parity is None: + raise DurableChangeTrainError("released train lacks historical schema evidence for forward-version admission") + historical = train.proof.fresh_ddl_parity + if ( + historical.tier is not train.tier + or historical.target_version != train.target_version + or historical.migrated_version != train.target_version + or historical.fresh_version != train.target_version + or not historical.matches + or historical.missing_objects + or historical.unexpected_objects + or historical.changed_objects + or historical.migrated_inventory_sha256 != train.apply_evidence.post.schema_inventory_sha256 + or historical.fresh_inventory_sha256 != train.fresh_ddl_parity.fresh_inventory_sha256 + ): + raise DurableChangeTrainError( + "released train lacks exact historical schema evidence for forward-version admission" + ) + return historical + + +def _verify_released_train_live_tier( + archive_root: Path, + conn: sqlite3.Connection, + train: DurableChangeTrain, + *, + current_target_version: int | None = None, + actual_evidence: DurableDatabaseEvidence | None = None, +) -> DurableForwardVersionReceipt | None: """Verify a released train remains represented after later trains advance it.""" if train.apply_evidence is None: raise DurableChangeTrainError(f"{train.state.value} train lacks post-apply continuity evidence") - actual = capture_durable_database_evidence(conn, train.tier) + actual = actual_evidence or capture_durable_database_evidence(conn, train.tier) if actual.user_version < train.target_version: raise DurableChangeTrainError( f"{train.tier.value} durable tier continuity proof failed: live version regressed below released train " @@ -1399,13 +1448,38 @@ def _verify_released_train_live_tier(archive_root: Path, conn: sqlite3.Connectio label="source continuity refresh", ) else: - _verify_persisted_live_tier_continuity(conn, train) - return - integrity = conn.execute("PRAGMA integrity_check").fetchone() - if integrity != ("ok",): + _verify_persisted_live_tier_continuity(conn, train, actual=actual) + return None + historical = _historical_schema_evidence(train) + expected_identity = train.apply_evidence.post.archive_identity_digest + if actual.archive_identity_digest != expected_identity: + raise DurableChangeTrainError( + f"{train.tier.value} durable tier immutable archive identity differs from historical train " + f"v{train.target_version} after later train advancement" + ) + if actual.quick_check != ("ok",): raise DurableChangeTrainError( f"{train.tier.value} durable tier integrity check failed after later train advancement" ) + runtime_target = ( + cast(dict[ArchiveTier, int], vars(_migration_runner)["ARCHIVE_VERSION_BY_TIER"])[train.tier] + if current_target_version is None + else current_target_version + ) + if actual.user_version > runtime_target: + raise DurableChangeTrainError( + f"{train.tier.value} durable tier version {actual.user_version} is newer than current target " + f"v{runtime_target}; historical train v{train.target_version} cannot admit it" + ) + return DurableForwardVersionReceipt( + tier=train.tier, + historical_train_id=train.train_id, + historical_target_version=train.target_version, + current_target_version=runtime_target, + observed_live_version=actual.user_version, + historical_schema_inventory_sha256=historical.migrated_inventory_sha256, + archive_identity_digest=actual.archive_identity_digest, + ) def _prove_and_release_persisted_train( @@ -1536,8 +1610,18 @@ def execute_durable_change_train( f"released {tier.value} train {train.train_id} expects live v{runtime_target_version}, " f"found v{live_version}; authorize a new execution" ) - _verify_released_train_live_tier(archive_root, live, train) - return DurableChangeTrainExecution(train=train, manifest_path=manifest_path, migration_result=None) + forward_version_receipt = _verify_released_train_live_tier( + archive_root, + live, + train, + current_target_version=runtime_target_version, + ) + return DurableChangeTrainExecution( + train=train, + manifest_path=manifest_path, + migration_result=None, + forward_version_receipt=forward_version_receipt, + ) if train.state is DurableChangeTrainState.DECLARED: previous_revision = train.revision @@ -1636,6 +1720,7 @@ def _reconcile_durable_change_train_startup_locked(archive_root: Path) -> tuple[ if not manifest_root.is_dir(): return () reconciled: list[Path] = [] + live_evidence_by_tier: dict[ArchiveTier, DurableDatabaseEvidence] = {} for manifest_path in sorted(manifest_root.glob("*.json")): train = load_durable_change_train_manifest(manifest_path) if train.state is DurableChangeTrainState.FAILED: @@ -1667,7 +1752,16 @@ def _reconcile_durable_change_train_startup_locked(archive_root: Path) -> tuple[ train = _persist_train_transition(manifest_path, recovered, expected_revision=train.revision) if train.state is DurableChangeTrainState.RELEASED: with _open_existing_tier(archive_root / f"{train.tier.value}.db") as live: - _verify_released_train_live_tier(archive_root, live, train) + actual = live_evidence_by_tier.get(train.tier) + if actual is None: + actual = capture_durable_database_evidence(live, train.tier) + live_evidence_by_tier[train.tier] = actual + _verify_released_train_live_tier( + archive_root, + live, + train, + actual_evidence=actual, + ) reconciled.append(manifest_path) continue if train.state not in { @@ -1697,6 +1791,7 @@ def __getattr__(name: str) -> object: "DurableChangeTrainManifest", "DurableMigrationSidecar", "DurableChangeTrainExecution", + "DurableForwardVersionReceipt", "durable_migration_sidecar_name", "validate_durable_migration_sidecars", "durable_change_train_policy_report", diff --git a/tests/unit/daemon/test_raw_failure_sample.py b/tests/unit/daemon/test_raw_failure_sample.py index 290876596d..0134f0a281 100644 --- a/tests/unit/daemon/test_raw_failure_sample.py +++ b/tests/unit/daemon/test_raw_failure_sample.py @@ -570,6 +570,45 @@ def test_daemon_status_lifecycle_counts_match_the_shared_projection(self, tmp_pa assert info["terminal_rejections"] == snapshot.terminal == 1 assert info["unexplained_failures"] == snapshot.unexplained == 0 + def test_lifecycle_samples_are_sql_bounded_for_large_file_backed_failure_sets( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + source_db = tmp_path / "source.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + with sqlite3.connect(source_db) as conn: + conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, + blob_size, acquired_at_ms, parse_error + ) VALUES (?, 'codex-session', ?, '/data/large-failures.jsonl', ?, ?, 0, ?, 'bad input') + """, + [ + (f"raw-{index}", f"native-{index}", index, bytes(32), 1_770_000_000_000 + index) + for index in range(200) + ], + ) + conn.commit() + + statements: list[str] = [] + + def open_traced_readonly(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(f"file:{path.resolve()}?mode=ro", uri=True) + connection.set_trace_callback(statements.append) + return connection + + monkeypatch.setattr("polylogue.storage.raw_failure_lifecycle.open_readonly_connection", open_traced_readonly) + snapshot = read_raw_failure_lifecycle(source_db, sample_limit=3) + + assert snapshot.parse_failures == snapshot.unexplained == 200 + assert len(snapshot.samples) == 3 + bounded_sample_queries = [ + statement for statement in statements if "FROM failed" in statement and "LIMIT 3" in statement + ] + assert len(bounded_sample_queries) == 1 + @pytest.mark.parametrize("source_state", ["missing", "malformed"]) def test_status_fails_closed_when_source_lifecycle_is_unavailable( self, diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 1a1a506d88..b16654bcf9 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -5,6 +5,7 @@ import hashlib import json import os +import shutil import sqlite3 import sys from collections.abc import Callable, Iterator @@ -1116,6 +1117,52 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( assert conn.execute("SELECT name FROM sqlite_schema WHERE name='later_items'").fetchone() == ("later_items",) assert released == [True, True] + historical_manifest = durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 2) + historical_train = load_durable_change_train_manifest(historical_manifest) + with sqlite3.connect(db_path) as conn: + actual = migration_runner.capture_durable_database_evidence(conn, ArchiveTier.SOURCE) + receipt = durable_change_train_module._verify_released_train_live_tier( + tmp_path, + conn, + historical_train, + current_target_version=3, + actual_evidence=actual, + ) + assert receipt is not None + assert receipt.historical_target_version == 2 + assert receipt.current_target_version == 3 + assert receipt.observed_live_version == 3 + assert historical_train.proof is not None + assert ( + receipt.historical_schema_inventory_sha256 == historical_train.proof.fresh_ddl_parity.migrated_inventory_sha256 + ) + + captures = 0 + real_capture = migration_runner.capture_durable_database_evidence + + def count_captures(connection: sqlite3.Connection, tier: ArchiveTier) -> migration_runner.DurableDatabaseEvidence: + nonlocal captures + captures += 1 + return real_capture(connection, tier) + + monkeypatch.setattr(durable_change_train_module, "capture_durable_database_evidence", count_captures) + assert reconcile_durable_change_train_startup(tmp_path) == ( + durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 2), + durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 3), + ) + assert captures == 1 + + unrelated_root = tmp_path / "unrelated-archive" + unrelated_root.mkdir() + shutil.copy2(db_path, unrelated_root / "source.db") + unrelated_manifest = durable_change_train_manifest_path(unrelated_root, ArchiveTier.SOURCE, 2) + unrelated_manifest.parent.mkdir(parents=True) + shutil.copy2(historical_manifest, unrelated_manifest) + with sqlite3.connect(unrelated_root / "source.db") as conn: + assert conn.execute("PRAGMA integrity_check").fetchone() == ("ok",) + with pytest.raises(DurableChangeTrainError, match="immutable archive identity differs"): + reconcile_durable_change_train_startup(unrelated_root) + def test_future_train_sidecar_hash_and_slot_are_admission_bound( tmp_path: Path, monkeypatch: pytest.MonkeyPatch From 4d27f474532cccb768e21f7eb4e1db40d50fddd9 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 17:44:07 +0200 Subject: [PATCH 02/32] chore: refresh PR scope authority Reissue the exact carrier head after rebasing onto current master. From e41daa70f62e468f795aa02d2e2bdd449762ee4e Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 18:46:18 +0200 Subject: [PATCH 03/32] fix(storage): bind forward train admission to schema objects Problem: forward admission validated a released tier's version, quick check, and aggregate schema digest, but did not prove that historical objects still existed with their original definitions after later train advancement. What changed: reconstruct the released train's canonical historical inventory from the package DDL, require every historical object and definition to remain present in the live tier, and add a regression that rejects a dropped table. Compatibility/migration: this strengthens admission and startup validation for durable tiers. It does not alter migration SQL or mutate live archives. Ref polylogue-6k0na Co-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 37 +++++++++++++++++++ .../unit/storage/test_durable_change_train.py | 13 +++++++ 2 files changed, 50 insertions(+) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 9e1135956c..18c345f7d4 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -12,6 +12,7 @@ import sqlite3 import tempfile from collections.abc import Callable, Sequence +from contextlib import closing from dataclasses import dataclass, replace from importlib import resources from pathlib import Path @@ -1422,6 +1423,23 @@ def _historical_schema_evidence(train: DurableChangeTrain) -> DurableFreshDDLPar return historical +def _historical_schema_inventory(train: DurableChangeTrain) -> _migration_runner.DurableSchemaInventory: + """Reconstruct the exact canonical object set owned by a released train.""" + historical = _historical_schema_evidence(train) + with closing(sqlite3.connect(":memory:")) as fresh: + fresh.execute("PRAGMA foreign_keys = ON") + fresh.executescript(_migration_runner.ARCHIVE_DDL_BY_TIER[train.tier]) + fresh.execute(f"PRAGMA user_version = {train.target_version}") + _migration_runner._prepare_fresh_connection_for_target(fresh, train.tier, train.target_version) + fresh.commit() + inventory = _migration_runner.capture_durable_schema_inventory(fresh) + if inventory.sha256 != historical.fresh_inventory_sha256: + raise DurableChangeTrainError( + "released train canonical schema inventory no longer matches its historical fresh-DDL proof" + ) + return inventory + + def _verify_released_train_live_tier( archive_root: Path, conn: sqlite3.Connection, @@ -1451,6 +1469,7 @@ def _verify_released_train_live_tier( _verify_persisted_live_tier_continuity(conn, train, actual=actual) return None historical = _historical_schema_evidence(train) + expected_inventory = _historical_schema_inventory(train) expected_identity = train.apply_evidence.post.archive_identity_digest if actual.archive_identity_digest != expected_identity: raise DurableChangeTrainError( @@ -1461,6 +1480,24 @@ def _verify_released_train_live_tier( raise DurableChangeTrainError( f"{train.tier.value} durable tier integrity check failed after later train advancement" ) + live_inventory = _migration_runner.capture_durable_schema_inventory(conn) + if live_inventory.sha256 != actual.schema_inventory_sha256: + raise DurableChangeTrainError( + f"{train.tier.value} durable tier schema inventory changed during forward admission" + ) + expected_by_ref = {item.object_ref: item for item in expected_inventory.objects} + live_by_ref = {item.object_ref: item for item in live_inventory.objects} + missing = sorted(set(expected_by_ref) - set(live_by_ref)) + changed = sorted( + object_ref + for object_ref in set(expected_by_ref) & set(live_by_ref) + if expected_by_ref[object_ref].definition_sha256 != live_by_ref[object_ref].definition_sha256 + ) + if missing or changed: + raise DurableChangeTrainError( + f"{train.tier.value} durable tier historical schema objects changed after later train advancement: " + f"missing={missing}, changed={changed}" + ) runtime_target = ( cast(dict[ArchiveTier, int], vars(_migration_runner)["ARCHIVE_VERSION_BY_TIER"])[train.tier] if current_target_version is None diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index b16654bcf9..3cd1ed764d 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1163,6 +1163,19 @@ def count_captures(connection: sqlite3.Connection, tier: ArchiveTier) -> migrati with pytest.raises(DurableChangeTrainError, match="immutable archive identity differs"): reconcile_durable_change_train_startup(unrelated_root) + with sqlite3.connect(db_path) as conn: + conn.execute("DROP TABLE base_items") + conn.commit() + tampered = migration_runner.capture_durable_database_evidence(conn, ArchiveTier.SOURCE) + with pytest.raises(DurableChangeTrainError, match="historical schema objects changed"): + durable_change_train_module._verify_released_train_live_tier( + tmp_path, + conn, + historical_train, + current_target_version=3, + actual_evidence=tampered, + ) + def test_future_train_sidecar_hash_and_slot_are_admission_bound( tmp_path: Path, monkeypatch: pytest.MonkeyPatch From 0057a87bde5c62a7f579e27c2090d948921724c0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 18:48:45 +0200 Subject: [PATCH 04/32] fix(storage): read archive DDL through typed module boundary Problem: strict mypy rejects direct access to the migration runner's runtime DDL mapping because it is not part of that module's explicit export surface. What changed: read the existing runtime mapping through the module namespace and cast it to the established ArchiveTier-to-SQL type. Compatibility/migration: typing-only follow-up to the durable train admission check. Runtime behavior is unchanged. Co-Authored-By: Claude --- polylogue/storage/sqlite/durable_change_train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 18c345f7d4..432d319d79 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -1428,7 +1428,8 @@ def _historical_schema_inventory(train: DurableChangeTrain) -> _migration_runner historical = _historical_schema_evidence(train) with closing(sqlite3.connect(":memory:")) as fresh: fresh.execute("PRAGMA foreign_keys = ON") - fresh.executescript(_migration_runner.ARCHIVE_DDL_BY_TIER[train.tier]) + archive_ddl = cast(dict[ArchiveTier, str], vars(_migration_runner)["ARCHIVE_DDL_BY_TIER"]) + fresh.executescript(archive_ddl[train.tier]) fresh.execute(f"PRAGMA user_version = {train.target_version}") _migration_runner._prepare_fresh_connection_for_target(fresh, train.tier, train.target_version) fresh.commit() From 5524b84c8000e8216d8bfad0e4981074dff9ee7e Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 18:51:36 +0200 Subject: [PATCH 05/32] chore(ci): refresh durable train scope carrier Refresh the non-draft PR trigger after publishing the verified historical-schema admission repair. No product behavior changes.\n\nCo-Authored-By: Claude From 59c21ed4706ecee3637e6b8132c7b6420107ff8e Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 19:14:07 +0200 Subject: [PATCH 06/32] fix(storage): harden durable forward admission Problem: durable forward admission was coupled to rebuildable generation identity, skipped SQLite integrity checks, and did not expose a forward receipt when the live tier had already reached its target. Raw-failure lifecycle summaries and samples also came from separate read snapshots. What changed: bind durable evidence to source/user identities, run full integrity checks, compare the live schema with the exact canonical version, return the newest historical forward receipt, and derive lifecycle counts and bounded samples from one read transaction and one query. Tests cover schema drift, receipt reachability, lifecycle snapshot consistency, and sample bounds. Compatibility/migration: durable admission is stricter for schema drift and integrity failures. Existing valid additive migrations remain admitted against their canonical live version. Ref polylogue-dcrmm Co-Authored-By: Claude --- polylogue/storage/raw_failure_lifecycle.py | 41 +++++----- .../storage/sqlite/durable_change_train.py | 75 ++++++++++++++----- polylogue/storage/sqlite/migration_runner.py | 7 +- tests/unit/daemon/test_raw_failure_sample.py | 6 +- .../unit/storage/test_durable_change_train.py | 13 +++- 5 files changed, 101 insertions(+), 41 deletions(-) diff --git a/polylogue/storage/raw_failure_lifecycle.py b/polylogue/storage/raw_failure_lifecycle.py index fe1d198c68..5bad3b2349 100644 --- a/polylogue/storage/raw_failure_lifecycle.py +++ b/polylogue/storage/raw_failure_lifecycle.py @@ -111,6 +111,7 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra logger.warning("could not open source.db read-only", exc_info=exc) return RawFailureLifecycleSnapshot(False, reason=f"could not open source.db read-only: {exc}") try: + conn.execute("BEGIN") raw_table = conn.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'raw_sessions'" ).fetchone() @@ -152,36 +153,40 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra ORDER BY a.last_observed_at_ms DESC, a.artifact_id DESC LIMIT 1 ) AS support_status + ,r.acquired_at_ms FROM raw_sessions AS r WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') OR r.validation_status = 'failed' - ORDER BY r.acquired_at_ms DESC, r.raw_id DESC """ else: failure_query = """ - SELECT r.raw_id, r.origin, r.validation_status, NULL AS artifact_kind, NULL AS support_status + SELECT r.raw_id, r.origin, r.validation_status, NULL AS artifact_kind, NULL AS support_status, + r.acquired_at_ms FROM raw_sessions AS r WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') OR r.validation_status = 'failed' - ORDER BY r.acquired_at_ms DESC, r.raw_id DESC """ - summary_rows = conn.execute( + summary_counts: Counter[tuple[object, object, object, object]] = Counter() + sample_rows: list[tuple[object, ...]] = [] + sample_limit = max(0, sample_limit) + failure_rows = conn.execute( f""" - WITH failed AS ({failure_query}) - SELECT origin, validation_status, artifact_kind, support_status, COUNT(*) - FROM failed - GROUP BY origin, validation_status, artifact_kind, support_status + WITH failed AS ({failure_query}), + ranked AS ( + SELECT failed.*, + ROW_NUMBER() OVER (ORDER BY acquired_at_ms DESC, raw_id DESC) AS sample_rank + FROM failed + ) + SELECT raw_id, origin, validation_status, artifact_kind, support_status, sample_rank + FROM ranked """ - ).fetchall() - sample_rows = conn.execute( - f""" - WITH failed AS ({failure_query}) - SELECT raw_id, origin, validation_status, artifact_kind, support_status - FROM failed - LIMIT ? - """, - (max(0, sample_limit),), - ).fetchall() + ) + for row in failure_rows: + key = (row[1], row[2], row[3], row[4]) + summary_counts[key] += 1 + if int(row[5]) <= sample_limit: + sample_rows.append(tuple(row[:5])) + summary_rows = [(*key, count) for key, count in summary_counts.items()] except sqlite3.Error as exc: logger.warning("could not read raw failure lifecycle", exc_info=exc) return RawFailureLifecycleSnapshot(False, reason=f"could not read raw failure lifecycle: {exc}") diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 432d319d79..7a9957cb76 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -1423,22 +1423,16 @@ def _historical_schema_evidence(train: DurableChangeTrain) -> DurableFreshDDLPar return historical -def _historical_schema_inventory(train: DurableChangeTrain) -> _migration_runner.DurableSchemaInventory: - """Reconstruct the exact canonical object set owned by a released train.""" - historical = _historical_schema_evidence(train) +def _canonical_schema_inventory(tier: ArchiveTier, target_version: int) -> _migration_runner.DurableSchemaInventory: + """Construct the canonical object set for one live durable schema version.""" with closing(sqlite3.connect(":memory:")) as fresh: fresh.execute("PRAGMA foreign_keys = ON") archive_ddl = cast(dict[ArchiveTier, str], vars(_migration_runner)["ARCHIVE_DDL_BY_TIER"]) - fresh.executescript(archive_ddl[train.tier]) - fresh.execute(f"PRAGMA user_version = {train.target_version}") - _migration_runner._prepare_fresh_connection_for_target(fresh, train.tier, train.target_version) + fresh.executescript(archive_ddl[tier]) + fresh.execute(f"PRAGMA user_version = {target_version}") + _migration_runner._prepare_fresh_connection_for_target(fresh, tier, target_version) fresh.commit() - inventory = _migration_runner.capture_durable_schema_inventory(fresh) - if inventory.sha256 != historical.fresh_inventory_sha256: - raise DurableChangeTrainError( - "released train canonical schema inventory no longer matches its historical fresh-DDL proof" - ) - return inventory + return _migration_runner.capture_durable_schema_inventory(fresh) def _verify_released_train_live_tier( @@ -1470,7 +1464,6 @@ def _verify_released_train_live_tier( _verify_persisted_live_tier_continuity(conn, train, actual=actual) return None historical = _historical_schema_evidence(train) - expected_inventory = _historical_schema_inventory(train) expected_identity = train.apply_evidence.post.archive_identity_digest if actual.archive_identity_digest != expected_identity: raise DurableChangeTrainError( @@ -1481,23 +1474,30 @@ def _verify_released_train_live_tier( raise DurableChangeTrainError( f"{train.tier.value} durable tier integrity check failed after later train advancement" ) + integrity_check = tuple(str(row[0]) for row in conn.execute("PRAGMA integrity_check")) + if integrity_check != ("ok",): + raise DurableChangeTrainError( + f"{train.tier.value} durable tier integrity check failed after later train advancement: {integrity_check}" + ) live_inventory = _migration_runner.capture_durable_schema_inventory(conn) if live_inventory.sha256 != actual.schema_inventory_sha256: raise DurableChangeTrainError( f"{train.tier.value} durable tier schema inventory changed during forward admission" ) + expected_inventory = _canonical_schema_inventory(train.tier, actual.user_version) expected_by_ref = {item.object_ref: item for item in expected_inventory.objects} live_by_ref = {item.object_ref: item for item in live_inventory.objects} missing = sorted(set(expected_by_ref) - set(live_by_ref)) + unexpected = sorted(set(live_by_ref) - set(expected_by_ref)) changed = sorted( object_ref for object_ref in set(expected_by_ref) & set(live_by_ref) if expected_by_ref[object_ref].definition_sha256 != live_by_ref[object_ref].definition_sha256 ) - if missing or changed: + if missing or unexpected or changed: raise DurableChangeTrainError( - f"{train.tier.value} durable tier historical schema objects changed after later train advancement: " - f"missing={missing}, changed={changed}" + f"{train.tier.value} durable tier schema differs from the canonical live version: " + f"missing={missing}, unexpected={unexpected}, changed={changed}" ) runtime_target = ( cast(dict[ArchiveTier, int], vars(_migration_runner)["ARCHIVE_VERSION_BY_TIER"])[train.tier] @@ -1520,6 +1520,34 @@ def _verify_released_train_live_tier( ) +def _forward_version_receipt_for_current_tier( + archive_root: Path, + conn: sqlite3.Connection, + tier: ArchiveTier, + *, + current_version: int, + current_target_version: int, +) -> DurableForwardVersionReceipt | None: + """Return the newest released historical-train receipt at the live target.""" + manifest_root = archive_root / ".maintenance-state" / "durable-change-trains" + historical: list[DurableChangeTrain] = [] + if manifest_root.is_dir(): + for path in sorted(manifest_root.glob(f"{tier.value}-*.json")): + train = load_durable_change_train_manifest(path) + if train.state is DurableChangeTrainState.RELEASED and train.target_version < current_version: + historical.append(train) + for train in sorted(historical, key=lambda item: item.target_version, reverse=True): + receipt = _verify_released_train_live_tier( + archive_root, + conn, + train, + current_target_version=current_target_version, + ) + if receipt is not None: + return receipt + return None + + def _prove_and_release_persisted_train( archive_root: Path, manifest_path: Path, @@ -1630,7 +1658,20 @@ def execute_durable_change_train( f"durable migration chain for {tier.value} stops at v{current_version}; " f"runtime requires v{runtime_target_version} and the next train sidecar is missing" ) - return DurableChangeTrainExecution(train=None, manifest_path=None, migration_result=legacy_result) + with _open_existing_tier(tier_path) as live: + forward_version_receipt = _forward_version_receipt_for_current_tier( + archive_root, + live, + tier, + current_version=current_version, + current_target_version=runtime_target_version, + ) + return DurableChangeTrainExecution( + train=None, + manifest_path=None, + migration_result=legacy_result, + forward_version_receipt=forward_version_receipt, + ) manifest_path = durable_change_train_manifest_path(archive_root, tier, sidecar.slot) if manifest_path.exists(): diff --git a/polylogue/storage/sqlite/migration_runner.py b/polylogue/storage/sqlite/migration_runner.py index decd9e3a0d..338614b636 100644 --- a/polylogue/storage/sqlite/migration_runner.py +++ b/polylogue/storage/sqlite/migration_runner.py @@ -1607,7 +1607,12 @@ def capture_durable_database_evidence( live_path = _connection_main_path(conn) from polylogue.storage.archive_identity import ArchiveIdentity - archive_identity_digest = ArchiveIdentity.resolve(live_path.parent).authority_identity_digest + # Durable migration evidence must survive replacement of rebuildable + # generations. The source/user tier identities are the durable authority; + # active index, embeddings, and ops identities belong to derived/runtime + # state and must not invalidate a durable train. + durable_id = ArchiveIdentity.resolve(live_path.parent).durable_id + archive_identity_digest = hashlib.sha256(durable_id.encode("utf-8")).hexdigest() content_hasher = hashlib.sha256() for statement in conn.iterdump(): content_hasher.update(statement.encode("utf-8")) diff --git a/tests/unit/daemon/test_raw_failure_sample.py b/tests/unit/daemon/test_raw_failure_sample.py index 0134f0a281..0498603970 100644 --- a/tests/unit/daemon/test_raw_failure_sample.py +++ b/tests/unit/daemon/test_raw_failure_sample.py @@ -604,10 +604,8 @@ def open_traced_readonly(path: Path) -> sqlite3.Connection: assert snapshot.parse_failures == snapshot.unexplained == 200 assert len(snapshot.samples) == 3 - bounded_sample_queries = [ - statement for statement in statements if "FROM failed" in statement and "LIMIT 3" in statement - ] - assert len(bounded_sample_queries) == 1 + lifecycle_queries = [statement for statement in statements if "ROW_NUMBER() OVER" in statement] + assert len(lifecycle_queries) == 1 @pytest.mark.parametrize("source_state", ["missing", "malformed"]) def test_status_fails_closed_when_source_lifecycle_is_unavailable( diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 3cd1ed764d..d87c87cd65 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1116,6 +1116,17 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( assert conn.execute("PRAGMA user_version").fetchone() == (3,) assert conn.execute("SELECT name FROM sqlite_schema WHERE name='later_items'").fetchone() == ("later_items",) assert released == [True, True] + third = execute_durable_change_train( + tmp_path, + ArchiveTier.SOURCE, + backup_manifest=None, + daemon_stopped_evidence_ref="proof:daemon-stopped", + single_writer_evidence_ref="proof:archive-ownership-lock", + release_archive_ownership=lambda: released.append(True), + ) + assert third.forward_version_receipt is not None + assert third.forward_version_receipt.historical_target_version == 2 + assert third.forward_version_receipt.observed_live_version == 3 historical_manifest = durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 2) historical_train = load_durable_change_train_manifest(historical_manifest) @@ -1167,7 +1178,7 @@ def count_captures(connection: sqlite3.Connection, tier: ArchiveTier) -> migrati conn.execute("DROP TABLE base_items") conn.commit() tampered = migration_runner.capture_durable_database_evidence(conn, ArchiveTier.SOURCE) - with pytest.raises(DurableChangeTrainError, match="historical schema objects changed"): + with pytest.raises(DurableChangeTrainError, match="canonical live version"): durable_change_train_module._verify_released_train_live_tier( tmp_path, conn, From 4353c45b7dc8092003c60e6fbb7180e99b874ae0 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 19:38:13 +0200 Subject: [PATCH 07/32] fix(storage): admit legacy durable identity evidence Problem: Existing durable train manifests store the pre-split full-archive identity digest, while new captures use the durable-tier identity. Startup continuity would reject an unchanged archive before another migration.\n\nWhat changed: Accept a legacy digest only when it matches the current legacy archive identity and the fresh evidence matches the durable identity. Apply the compatibility check to rollback, source continuity, and forward-admission paths, with a regression test.\n\nCompatibility/migration: New evidence remains durable-tier bound. Legacy manifests are admitted only during the narrow transition case and still require matching schema, content, and integrity evidence.\n\nCo-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 20 +++++++++--- polylogue/storage/sqlite/migration_runner.py | 31 ++++++++++++++++++- .../unit/storage/test_durable_change_train.py | 19 ++++++++++++ 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 7a9957cb76..0d5096123a 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -37,6 +37,7 @@ DurableMigrationClaim, DurableRuntimeConsumerResult, MigrationResult, + _archive_identity_continuity_matches, _assert_durable_database_continuity, _canonical_json_sha256, _require_nonempty, @@ -859,6 +860,7 @@ def _refresh_released_source_train_continuity_locked( current, retained_current, label="source continuity retained refresh", + connection=connection, ) except DurableChangeTrainError: pass @@ -903,16 +905,25 @@ def _refresh_released_source_train_continuity_locked( pre_mutation_evidence, baseline, label="source continuity pre-mutation", + connection=connection, ) except DurableChangeTrainError as exc: raise DurableSourceContinuitySemanticError( "source continuity refresh pre-state contains unreceipted content drift" ) from exc - if pre_mutation_evidence.archive_identity_digest != train.apply_evidence.post.archive_identity_digest: + if not _archive_identity_continuity_matches( + pre_mutation_evidence.archive_identity_digest, + train.apply_evidence.post.archive_identity_digest, + archive_root, + ): raise DurableSourceContinuitySemanticError( "source continuity refresh pre-state has the wrong archive identity" ) - if current.archive_identity_digest != train.apply_evidence.post.archive_identity_digest: + if not _archive_identity_continuity_matches( + current.archive_identity_digest, + train.apply_evidence.post.archive_identity_digest, + archive_root, + ): raise DurableSourceContinuitySemanticError("source continuity refresh changed archive identity") if pre_mutation_evidence.quick_check != ("ok",) or current.quick_check != ("ok",): raise DurableSourceContinuitySemanticError( @@ -1393,7 +1404,7 @@ def _verify_persisted_live_tier_continuity( f"{train.tier.value} durable tier continuity proof failed; refusing startup initialization/release" ) try: - _assert_durable_database_continuity(actual, expected, label=train.tier.value) + _assert_durable_database_continuity(actual, expected, label=train.tier.value, connection=conn) except DurableChangeTrainError as exc: raise DurableChangeTrainError( f"{train.tier.value} durable tier continuity proof failed; refusing startup initialization/release" @@ -1459,13 +1470,14 @@ def _verify_released_train_live_tier( actual, train.source_continuity_evidence, label="source continuity refresh", + connection=conn, ) else: _verify_persisted_live_tier_continuity(conn, train, actual=actual) return None historical = _historical_schema_evidence(train) expected_identity = train.apply_evidence.post.archive_identity_digest - if actual.archive_identity_digest != expected_identity: + if not _archive_identity_continuity_matches(actual.archive_identity_digest, expected_identity, archive_root): raise DurableChangeTrainError( f"{train.tier.value} durable tier immutable archive identity differs from historical train " f"v{train.target_version} after later train advancement" diff --git a/polylogue/storage/sqlite/migration_runner.py b/polylogue/storage/sqlite/migration_runner.py index 338614b636..61cf7b054e 100644 --- a/polylogue/storage/sqlite/migration_runner.py +++ b/polylogue/storage/sqlite/migration_runner.py @@ -1629,18 +1629,46 @@ def capture_durable_database_evidence( ) +def _archive_identity_continuity_matches( + actual_digest: str, + expected_digest: str, + archive_root: Path, +) -> bool: + if actual_digest == expected_digest: + return True + from polylogue.storage.archive_identity import ArchiveIdentity + + identity = ArchiveIdentity.resolve(archive_root) + legacy_digest = identity.authority_identity_digest + durable_digest = hashlib.sha256(identity.durable_id.encode("utf-8")).hexdigest() + # Manifests written before the durable-tier identity split contain the + # old full-archive digest. Admit that legacy evidence only when the + # current archive still has the same legacy identity and the newly + # captured evidence proves the durable identity is unchanged. + return expected_digest == legacy_digest and actual_digest == durable_digest + + def _assert_durable_database_continuity( actual: DurableDatabaseEvidence, expected: DurableDatabaseEvidence, *, label: str, + connection: sqlite3.Connection | None = None, ) -> None: """Require the live durable file to retain its authenticated evidence.""" + identity_continuous = actual.archive_identity_digest == expected.archive_identity_digest + if not identity_continuous and connection is not None: + archive_root = _connection_main_path(connection).parent + identity_continuous = _archive_identity_continuity_matches( + actual.archive_identity_digest, + expected.archive_identity_digest, + archive_root, + ) if ( actual.quick_check != expected.quick_check or actual.quick_check != ("ok",) or actual.user_version != expected.user_version - or actual.archive_identity_digest != expected.archive_identity_digest + or not identity_continuous or actual.content_sha256 != expected.content_sha256 ): raise DurableChangeTrainError(f"{label} durable tier identity/content continuity proof failed") @@ -2373,6 +2401,7 @@ def recover_durable_change_train( capture_durable_database_evidence(conn, train.tier), pre, label="rolled-back recovery", + connection=conn, ) updated = replace( train, diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index d87c87cd65..4709dd474d 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1188,6 +1188,25 @@ def count_captures(connection: sqlite3.Connection, tier: ArchiveTier) -> migrati ) +def test_continuity_admits_legacy_full_archive_identity_digest(tmp_path: Path) -> None: + from polylogue.storage.archive_identity import ArchiveIdentity + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + with sqlite3.connect(tmp_path / "source.db") as conn: + current = migration_runner.capture_durable_database_evidence(conn, ArchiveTier.SOURCE) + legacy = replace( + current, + archive_identity_digest=ArchiveIdentity.resolve(tmp_path).authority_identity_digest, + ) + migration_runner._assert_durable_database_continuity( + current, + legacy, + label="legacy identity compatibility", + connection=conn, + ) + + def test_future_train_sidecar_hash_and_slot_are_admission_bound( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 541a8f2401c7021d26feed25ba6aed6dd96d8df5 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 20:13:42 +0200 Subject: [PATCH 08/32] fix(storage): bound raw failure lifecycle queries Problem: Lifecycle summaries scanned every failed row into Python and capped samples after materialization, making the read-only status path scale with an unbounded in-memory result. What changed: Group failure lifecycle evidence in SQL and apply the sample limit in SQL inside the existing read transaction. The focused test now checks both the aggregate and bounded sample statements. Compatibility/migration: Snapshot fields and lifecycle classification remain unchanged. --- polylogue/storage/raw_failure_lifecycle.py | 100 +++++++++---------- tests/unit/daemon/test_raw_failure_sample.py | 6 +- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/polylogue/storage/raw_failure_lifecycle.py b/polylogue/storage/raw_failure_lifecycle.py index 5bad3b2349..b1d733d7d3 100644 --- a/polylogue/storage/raw_failure_lifecycle.py +++ b/polylogue/storage/raw_failure_lifecycle.py @@ -130,63 +130,59 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'raw_artifacts'").fetchone() is not None ) + sample_limit = max(0, sample_limit) if has_artifacts: - failure_query = """ - SELECT r.raw_id, r.origin, r.validation_status, - ( - SELECT a.artifact_kind - FROM raw_artifacts AS a - WHERE a.raw_id = r.raw_id - AND a.origin = r.origin - AND a.source_path = r.source_path - AND a.source_index = r.source_index - ORDER BY a.last_observed_at_ms DESC, a.artifact_id DESC - LIMIT 1 - ) AS artifact_kind - ,( - SELECT a.support_status - FROM raw_artifacts AS a - WHERE a.raw_id = r.raw_id - AND a.origin = r.origin - AND a.source_path = r.source_path - AND a.source_index = r.source_index - ORDER BY a.last_observed_at_ms DESC, a.artifact_id DESC - LIMIT 1 - ) AS support_status - ,r.acquired_at_ms - FROM raw_sessions AS r - WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') - OR r.validation_status = 'failed' + failed_cte = """ + WITH latest_artifact AS ( + SELECT raw_id, origin, source_path, source_index, artifact_kind, support_status, + ROW_NUMBER() OVER ( + PARTITION BY raw_id, origin, source_path, source_index + ORDER BY last_observed_at_ms DESC, artifact_id DESC + ) AS artifact_rank + FROM raw_artifacts + ), failed AS ( + SELECT r.raw_id, r.origin, r.validation_status, + a.artifact_kind, a.support_status, r.acquired_at_ms + FROM raw_sessions AS r + LEFT JOIN latest_artifact AS a + ON a.raw_id = r.raw_id + AND a.origin = r.origin + AND a.source_path = r.source_path + AND a.source_index = r.source_index + AND a.artifact_rank = 1 + WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') + OR r.validation_status = 'failed' + ) """ else: - failure_query = """ - SELECT r.raw_id, r.origin, r.validation_status, NULL AS artifact_kind, NULL AS support_status, - r.acquired_at_ms - FROM raw_sessions AS r - WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') - OR r.validation_status = 'failed' - """ - summary_counts: Counter[tuple[object, object, object, object]] = Counter() - sample_rows: list[tuple[object, ...]] = [] - sample_limit = max(0, sample_limit) - failure_rows = conn.execute( - f""" - WITH failed AS ({failure_query}), - ranked AS ( - SELECT failed.*, - ROW_NUMBER() OVER (ORDER BY acquired_at_ms DESC, raw_id DESC) AS sample_rank - FROM failed + failed_cte = """ + WITH failed AS ( + SELECT r.raw_id, r.origin, r.validation_status, + NULL AS artifact_kind, NULL AS support_status, r.acquired_at_ms + FROM raw_sessions AS r + WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') + OR r.validation_status = 'failed' ) - SELECT raw_id, origin, validation_status, artifact_kind, support_status, sample_rank - FROM ranked """ - ) - for row in failure_rows: - key = (row[1], row[2], row[3], row[4]) - summary_counts[key] += 1 - if int(row[5]) <= sample_limit: - sample_rows.append(tuple(row[:5])) - summary_rows = [(*key, count) for key, count in summary_counts.items()] + summary_rows = conn.execute( + failed_cte + + """ + SELECT origin, validation_status, artifact_kind, support_status, COUNT(*) AS failure_count + FROM failed + GROUP BY origin, validation_status, artifact_kind, support_status + ORDER BY origin, validation_status, artifact_kind, support_status + """ + ).fetchall() + sample_rows = conn.execute( + failed_cte + + """ + SELECT raw_id, origin, validation_status, artifact_kind, support_status + FROM failed + ORDER BY acquired_at_ms DESC, raw_id DESC + LIMIT ? + """, + (sample_limit,), + ).fetchall() except sqlite3.Error as exc: logger.warning("could not read raw failure lifecycle", exc_info=exc) return RawFailureLifecycleSnapshot(False, reason=f"could not read raw failure lifecycle: {exc}") diff --git a/tests/unit/daemon/test_raw_failure_sample.py b/tests/unit/daemon/test_raw_failure_sample.py index 0498603970..4c65a2b121 100644 --- a/tests/unit/daemon/test_raw_failure_sample.py +++ b/tests/unit/daemon/test_raw_failure_sample.py @@ -604,8 +604,10 @@ def open_traced_readonly(path: Path) -> sqlite3.Connection: assert snapshot.parse_failures == snapshot.unexplained == 200 assert len(snapshot.samples) == 3 - lifecycle_queries = [statement for statement in statements if "ROW_NUMBER() OVER" in statement] - assert len(lifecycle_queries) == 1 + summary_queries = [statement for statement in statements if "GROUP BY origin" in statement] + sample_queries = [statement for statement in statements if "LIMIT 3" in statement] + assert len(summary_queries) == 1 + assert len(sample_queries) == 1 @pytest.mark.parametrize("source_state", ["missing", "malformed"]) def test_status_fails_closed_when_source_lifecycle_is_unavailable( From f809a5366eee3392302c0a198c062d70c58a3724 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 20:15:24 +0200 Subject: [PATCH 09/32] fix(storage): cache durable train schema checks Problem: Startup reconciliation repeated full integrity scans and schema inventory construction for every released historical train in the same durable tier. What changed: Reuse per-tier integrity and schema inventories during startup reconciliation, and validate canonical inventory target versions and DDL registration before constructing them. Compatibility/migration: Direct forward-admission callers retain the same checks; only repeated startup observations are cached. --- .../storage/sqlite/durable_change_train.py | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 0d5096123a..de23d6de19 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -1436,12 +1436,21 @@ def _historical_schema_evidence(train: DurableChangeTrain) -> DurableFreshDDLPar def _canonical_schema_inventory(tier: ArchiveTier, target_version: int) -> _migration_runner.DurableSchemaInventory: """Construct the canonical object set for one live durable schema version.""" + try: + normalized_target_version = int(target_version) + except (TypeError, ValueError) as exc: + raise DurableChangeTrainError("canonical schema inventory target version must be an integer") from exc + if isinstance(target_version, bool): + raise DurableChangeTrainError("canonical schema inventory target version must be an integer") + try: + archive_ddl = _migration_runner.ARCHIVE_DDL_BY_TIER[tier] + except KeyError as exc: + raise DurableChangeTrainError(f"no canonical archive DDL is registered for {tier.value}") from exc with closing(sqlite3.connect(":memory:")) as fresh: fresh.execute("PRAGMA foreign_keys = ON") - archive_ddl = cast(dict[ArchiveTier, str], vars(_migration_runner)["ARCHIVE_DDL_BY_TIER"]) - fresh.executescript(archive_ddl[tier]) - fresh.execute(f"PRAGMA user_version = {target_version}") - _migration_runner._prepare_fresh_connection_for_target(fresh, tier, target_version) + fresh.executescript(archive_ddl) + fresh.execute(f"PRAGMA user_version = {normalized_target_version}") + _migration_runner._prepare_fresh_connection_for_target(fresh, tier, normalized_target_version) fresh.commit() return _migration_runner.capture_durable_schema_inventory(fresh) @@ -1453,6 +1462,9 @@ def _verify_released_train_live_tier( *, current_target_version: int | None = None, actual_evidence: DurableDatabaseEvidence | None = None, + integrity_check: tuple[str, ...] | None = None, + live_inventory: _migration_runner.DurableSchemaInventory | None = None, + canonical_inventory: _migration_runner.DurableSchemaInventory | None = None, ) -> DurableForwardVersionReceipt | None: """Verify a released train remains represented after later trains advance it.""" if train.apply_evidence is None: @@ -1486,17 +1498,17 @@ def _verify_released_train_live_tier( raise DurableChangeTrainError( f"{train.tier.value} durable tier integrity check failed after later train advancement" ) - integrity_check = tuple(str(row[0]) for row in conn.execute("PRAGMA integrity_check")) - if integrity_check != ("ok",): + observed_integrity = integrity_check or tuple(str(row[0]) for row in conn.execute("PRAGMA integrity_check")) + if observed_integrity != ("ok",): raise DurableChangeTrainError( - f"{train.tier.value} durable tier integrity check failed after later train advancement: {integrity_check}" + f"{train.tier.value} durable tier integrity check failed after later train advancement: {observed_integrity}" ) - live_inventory = _migration_runner.capture_durable_schema_inventory(conn) + live_inventory = live_inventory or _migration_runner.capture_durable_schema_inventory(conn) if live_inventory.sha256 != actual.schema_inventory_sha256: raise DurableChangeTrainError( f"{train.tier.value} durable tier schema inventory changed during forward admission" ) - expected_inventory = _canonical_schema_inventory(train.tier, actual.user_version) + expected_inventory = canonical_inventory or _canonical_schema_inventory(train.tier, actual.user_version) expected_by_ref = {item.object_ref: item for item in expected_inventory.objects} live_by_ref = {item.object_ref: item for item in live_inventory.objects} missing = sorted(set(expected_by_ref) - set(live_by_ref)) @@ -1812,6 +1824,9 @@ def _reconcile_durable_change_train_startup_locked(archive_root: Path) -> tuple[ return () reconciled: list[Path] = [] live_evidence_by_tier: dict[ArchiveTier, DurableDatabaseEvidence] = {} + live_integrity_by_tier: dict[ArchiveTier, tuple[str, ...]] = {} + live_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} + canonical_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} for manifest_path in sorted(manifest_root.glob("*.json")): train = load_durable_change_train_manifest(manifest_path) if train.state is DurableChangeTrainState.FAILED: @@ -1847,11 +1862,24 @@ def _reconcile_durable_change_train_startup_locked(archive_root: Path) -> tuple[ if actual is None: actual = capture_durable_database_evidence(live, train.tier) live_evidence_by_tier[train.tier] = actual + if train.tier not in live_integrity_by_tier: + live_integrity_by_tier[train.tier] = tuple( + str(row[0]) for row in live.execute("PRAGMA integrity_check") + ) + if train.tier not in live_inventory_by_tier: + live_inventory_by_tier[train.tier] = _migration_runner.capture_durable_schema_inventory(live) + if train.tier not in canonical_inventory_by_tier: + canonical_inventory_by_tier[train.tier] = _canonical_schema_inventory( + train.tier, actual.user_version + ) _verify_released_train_live_tier( archive_root, live, train, actual_evidence=actual, + integrity_check=live_integrity_by_tier[train.tier], + live_inventory=live_inventory_by_tier[train.tier], + canonical_inventory=canonical_inventory_by_tier[train.tier], ) reconciled.append(manifest_path) continue From 65eab44c817228c3c7c8db320e3a28c736482929 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 20:16:17 +0200 Subject: [PATCH 10/32] fix(storage): use exported durable DDL registry Problem: The canonical schema helper reached through a private module attribute that strict mypy rejects. What changed: Import the archive-tier DDL registry through its exported package boundary. Compatibility/migration: No schema or runtime behavior changes. --- polylogue/storage/sqlite/durable_change_train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index de23d6de19..743ae62573 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -23,6 +23,7 @@ BlobRefLivenessCandidateDigest, ) from polylogue.storage.sqlite import migration_runner as _migration_runner +from polylogue.storage.sqlite.archive_tiers import ARCHIVE_DDL_BY_TIER from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.migration_runner import ( DURABLE_CHANGE_TRAIN_FORMAT, @@ -1443,7 +1444,7 @@ def _canonical_schema_inventory(tier: ArchiveTier, target_version: int) -> _migr if isinstance(target_version, bool): raise DurableChangeTrainError("canonical schema inventory target version must be an integer") try: - archive_ddl = _migration_runner.ARCHIVE_DDL_BY_TIER[tier] + archive_ddl = ARCHIVE_DDL_BY_TIER[tier] except KeyError as exc: raise DurableChangeTrainError(f"no canonical archive DDL is registered for {tier.value}") from exc with closing(sqlite3.connect(":memory:")) as fresh: From cbfa55b1d17e043b2f473c0a43b41f0fb6a6b324 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 20:43:30 +0200 Subject: [PATCH 11/32] fix(storage): defer durable train admission scans Problem: equal-version durable train reconciliation performed integrity and schema scans that are only required when a live tier has advanced, and schema inventory lookup bypassed runtime registry overrides.\n\nWhat changed: defer forward-admission evidence scans until the live version exceeds the released target, and resolve the canonical DDL registry from migration_runner at call time.\n\nVerification: devtools test tests/unit/storage/test_durable_change_train.py tests/unit/daemon/test_raw_failure_sample.py (79 passed). --- .../storage/sqlite/durable_change_train.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 743ae62573..6379d926b5 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -23,7 +23,6 @@ BlobRefLivenessCandidateDigest, ) from polylogue.storage.sqlite import migration_runner as _migration_runner -from polylogue.storage.sqlite.archive_tiers import ARCHIVE_DDL_BY_TIER from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.migration_runner import ( DURABLE_CHANGE_TRAIN_FORMAT, @@ -1443,10 +1442,10 @@ def _canonical_schema_inventory(tier: ArchiveTier, target_version: int) -> _migr raise DurableChangeTrainError("canonical schema inventory target version must be an integer") from exc if isinstance(target_version, bool): raise DurableChangeTrainError("canonical schema inventory target version must be an integer") - try: - archive_ddl = ARCHIVE_DDL_BY_TIER[tier] - except KeyError as exc: - raise DurableChangeTrainError(f"no canonical archive DDL is registered for {tier.value}") from exc + registry = getattr(_migration_runner, "ARCHIVE_DDL_BY_TIER", None) + archive_ddl = registry.get(tier) if isinstance(registry, dict) else None + if not isinstance(archive_ddl, str): + raise DurableChangeTrainError(f"no canonical archive DDL is registered for {tier.value}") with closing(sqlite3.connect(":memory:")) as fresh: fresh.execute("PRAGMA foreign_keys = ON") fresh.executescript(archive_ddl) @@ -1863,24 +1862,25 @@ def _reconcile_durable_change_train_startup_locked(archive_root: Path) -> tuple[ if actual is None: actual = capture_durable_database_evidence(live, train.tier) live_evidence_by_tier[train.tier] = actual - if train.tier not in live_integrity_by_tier: - live_integrity_by_tier[train.tier] = tuple( - str(row[0]) for row in live.execute("PRAGMA integrity_check") - ) - if train.tier not in live_inventory_by_tier: - live_inventory_by_tier[train.tier] = _migration_runner.capture_durable_schema_inventory(live) - if train.tier not in canonical_inventory_by_tier: - canonical_inventory_by_tier[train.tier] = _canonical_schema_inventory( - train.tier, actual.user_version - ) + if actual.user_version > train.target_version: + if train.tier not in live_integrity_by_tier: + live_integrity_by_tier[train.tier] = tuple( + str(row[0]) for row in live.execute("PRAGMA integrity_check") + ) + if train.tier not in live_inventory_by_tier: + live_inventory_by_tier[train.tier] = _migration_runner.capture_durable_schema_inventory(live) + if train.tier not in canonical_inventory_by_tier: + canonical_inventory_by_tier[train.tier] = _canonical_schema_inventory( + train.tier, actual.user_version + ) _verify_released_train_live_tier( archive_root, live, train, actual_evidence=actual, - integrity_check=live_integrity_by_tier[train.tier], - live_inventory=live_inventory_by_tier[train.tier], - canonical_inventory=canonical_inventory_by_tier[train.tier], + integrity_check=live_integrity_by_tier.get(train.tier), + live_inventory=live_inventory_by_tier.get(train.tier), + canonical_inventory=canonical_inventory_by_tier.get(train.tier), ) reconciled.append(manifest_path) continue From b0ed05d8753c8a3659175cba029dffb1de40890a Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 20:48:39 +0200 Subject: [PATCH 12/32] chore(ci): refresh durable train carrier Refresh the non-draft PR validation boundary after the durable train review fixes. The commit has no source change; it exists to run checks against the current scope carrier. From 3b0352276ae56a4e6bcff4c327519a6c39d8ec3b Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 21:10:31 +0200 Subject: [PATCH 13/32] fix(storage): expose and reuse forward receipts Problem: no-op durable maintenance produced a forward-version receipt that was invisible on the CLI, and the same live evidence was rescanned after startup reconciliation.\n\nWhat changed: expose the typed receipt in JSON and plain maintenance output, and pass startup-captured evidence into the no-op admission path. Add CLI and scan-reuse regression coverage.\n\nVerification: devtools test tests/unit/storage/test_durable_change_train.py tests/unit/daemon/test_raw_failure_sample.py tests/unit/cli/test_archive_maintenance_cli.py::test_migrate_tier_cli_executes_and_persists_a_future_change_train tests/unit/cli/test_archive_maintenance_cli.py::test_migrate_tier_cli_exposes_forward_version_receipt (81 passed).\n\nResidual: two existing rebuild-index CLI tests remain red independently of this change: test_rebuild_index_byte_budget_defers_then_reaches_terminal_ready_candidate and test_rebuild_index_deadline_defers_postflight_until_resume. --- .../cli/commands/maintenance/_migrate_tier.py | 21 +++++++ .../storage/sqlite/durable_change_train.py | 53 ++++++++++++++-- .../unit/cli/test_archive_maintenance_cli.py | 63 +++++++++++++++++++ .../unit/storage/test_durable_change_train.py | 34 ++++++++++ 4 files changed, 167 insertions(+), 4 deletions(-) diff --git a/polylogue/cli/commands/maintenance/_migrate_tier.py b/polylogue/cli/commands/maintenance/_migrate_tier.py index 7859bdb25c..8efa19219b 100644 --- a/polylogue/cli/commands/maintenance/_migrate_tier.py +++ b/polylogue/cli/commands/maintenance/_migrate_tier.py @@ -138,6 +138,19 @@ def migrate_tier_command( "from_version": result.from_version if result is not None else 0 if initialized else None, "to_version": result.to_version if result is not None else initialized_version, "applied_versions": list(result.applied_versions) if result is not None else [], + "forward_version_receipt": ( + { + "tier": execution.forward_version_receipt.tier.value, + "historical_train_id": execution.forward_version_receipt.historical_train_id, + "historical_target_version": execution.forward_version_receipt.historical_target_version, + "current_target_version": execution.forward_version_receipt.current_target_version, + "observed_live_version": execution.forward_version_receipt.observed_live_version, + "historical_schema_inventory_sha256": execution.forward_version_receipt.historical_schema_inventory_sha256, + "archive_identity_digest": execution.forward_version_receipt.archive_identity_digest, + } + if execution is not None and execution.forward_version_receipt is not None + else None + ), } if output_format == "json": click.echo(json.dumps(payload, indent=2, sort_keys=True)) @@ -147,6 +160,14 @@ def migrate_tier_command( click.echo(f"Initialized missing {tier} tier at schema version {initialized_version}.") return if result is None: + if execution is not None and execution.forward_version_receipt is not None: + receipt = execution.forward_version_receipt + click.echo( + f"No pending durable migration for {tier}; historical train {receipt.historical_train_id} " + f"is admitted at live schema v{receipt.observed_live_version} " + f"(target v{receipt.current_target_version})." + ) + return click.echo(f"No pending durable migration for {tier}.") return applied = ", ".join(str(version) for version in result.applied_versions) or "none" diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 6379d926b5..8526cb113d 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -117,6 +117,16 @@ class DurableForwardVersionReceipt: archive_identity_digest: str +@dataclass(frozen=True, slots=True) +class _DurableForwardVersionEvidence: + """Cached live evidence reused by one no-op maintenance execution.""" + + actual: DurableDatabaseEvidence + integrity_check: tuple[str, ...] + live_inventory: _migration_runner.DurableSchemaInventory + canonical_inventory: _migration_runner.DurableSchemaInventory + + def durable_migration_sidecar_name(slot: int) -> str: """Return the only accepted Git path for a numbered train sidecar.""" if slot < 1: @@ -1551,6 +1561,7 @@ def _forward_version_receipt_for_current_tier( *, current_version: int, current_target_version: int, + evidence: _DurableForwardVersionEvidence | None = None, ) -> DurableForwardVersionReceipt | None: """Return the newest released historical-train receipt at the live target.""" manifest_root = archive_root / ".maintenance-state" / "durable-change-trains" @@ -1560,12 +1571,26 @@ def _forward_version_receipt_for_current_tier( train = load_durable_change_train_manifest(path) if train.state is DurableChangeTrainState.RELEASED and train.target_version < current_version: historical.append(train) + if not historical: + return None + if evidence is None: + actual = capture_durable_database_evidence(conn, tier) + evidence = _DurableForwardVersionEvidence( + actual=actual, + integrity_check=tuple(str(row[0]) for row in conn.execute("PRAGMA integrity_check")), + live_inventory=_migration_runner.capture_durable_schema_inventory(conn), + canonical_inventory=_canonical_schema_inventory(tier, actual.user_version), + ) for train in sorted(historical, key=lambda item: item.target_version, reverse=True): receipt = _verify_released_train_live_tier( archive_root, conn, train, current_target_version=current_target_version, + actual_evidence=evidence.actual, + integrity_check=evidence.integrity_check, + live_inventory=evidence.live_inventory, + canonical_inventory=evidence.canonical_inventory, ) if receipt is not None: return receipt @@ -1639,7 +1664,8 @@ def execute_durable_change_train( release_archive_ownership: Callable[[], None], ) -> DurableChangeTrainExecution: """Execute the real maintenance route through every persisted train state.""" - reconcile_durable_change_train_startup(archive_root) + forward_version_evidence: dict[ArchiveTier, _DurableForwardVersionEvidence] = {} + reconcile_durable_change_train_startup(archive_root, live_evidence_cache=forward_version_evidence) tier_path = archive_root / f"{tier.value}.db" with _open_existing_tier(tier_path) as probe: current_version = int(probe.execute("PRAGMA user_version").fetchone()[0] or 0) @@ -1689,6 +1715,7 @@ def execute_durable_change_train( tier, current_version=current_version, current_target_version=runtime_target_version, + evidence=forward_version_evidence.get(tier), ) return DurableChangeTrainExecution( train=None, @@ -1804,7 +1831,11 @@ def execute_durable_change_train( return DurableChangeTrainExecution(train=train, manifest_path=manifest_path, migration_result=migration_result) -def reconcile_durable_change_train_startup(archive_root: Path) -> tuple[Path, ...]: +def reconcile_durable_change_train_startup( + archive_root: Path, + *, + live_evidence_cache: dict[ArchiveTier, _DurableForwardVersionEvidence] | None = None, +) -> tuple[Path, ...]: """Reconcile backup-authorized trains left by a crashed maintenance process.""" from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation @@ -1813,10 +1844,17 @@ def reconcile_durable_change_train_startup(archive_root: Path) -> tuple[Path, .. owner_id=f"durable-train-recovery:{os.getpid()}", allow_reentrant=True, ): - return _reconcile_durable_change_train_startup_locked(archive_root) + return _reconcile_durable_change_train_startup_locked( + archive_root, + live_evidence_cache=live_evidence_cache, + ) -def _reconcile_durable_change_train_startup_locked(archive_root: Path) -> tuple[Path, ...]: +def _reconcile_durable_change_train_startup_locked( + archive_root: Path, + *, + live_evidence_cache: dict[ArchiveTier, _DurableForwardVersionEvidence] | None = None, +) -> tuple[Path, ...]: """Reconcile persisted trains while the caller holds archive ownership.""" _recover_pending_source_continuity_intents(archive_root) manifest_root = archive_root / ".maintenance-state" / "durable-change-trains" @@ -1873,6 +1911,13 @@ def _reconcile_durable_change_train_startup_locked(archive_root: Path) -> tuple[ canonical_inventory_by_tier[train.tier] = _canonical_schema_inventory( train.tier, actual.user_version ) + if live_evidence_cache is not None: + live_evidence_cache[train.tier] = _DurableForwardVersionEvidence( + actual=actual, + integrity_check=live_integrity_by_tier[train.tier], + live_inventory=live_inventory_by_tier[train.tier], + canonical_inventory=canonical_inventory_by_tier[train.tier], + ) _verify_released_train_live_tier( archive_root, live, diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index b023bbec92..a550d7773c 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -1535,6 +1535,69 @@ def test_migrate_tier_cli_executes_and_persists_a_future_change_train( assert conn.execute("SELECT name FROM sqlite_schema WHERE name='future_items'").fetchone() == ("future_items",) +def test_migrate_tier_cli_exposes_forward_version_receipt( + cli_workspace: dict[str, Path], + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from polylogue.cli.commands.maintenance import _migrate_tier + from polylogue.storage.sqlite.durable_change_train import ( + DurableChangeTrainExecution, + DurableForwardVersionReceipt, + ) + + source_db = cli_workspace["archive_root"] / "source.db" + if not source_db.exists(): + with sqlite3.connect(source_db) as conn: + conn.execute("PRAGMA user_version = 3") + conn.commit() + receipt = DurableForwardVersionReceipt( + tier=ArchiveTier.SOURCE, + historical_train_id="train:source:v2", + historical_target_version=2, + current_target_version=3, + observed_live_version=3, + historical_schema_inventory_sha256="a" * 64, + archive_identity_digest="b" * 64, + ) + monkeypatch.setattr( + _migrate_tier, + "execute_durable_change_train", + lambda *_args, **_kwargs: DurableChangeTrainExecution( + train=None, + manifest_path=None, + migration_result=None, + forward_version_receipt=receipt, + ), + ) + + result = cli_runner.invoke( + cli, + ["--plain", "ops", "maintenance", "migrate-tier", "source", "--output-format", "json"], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["forward_version_receipt"] == { + "archive_identity_digest": "b" * 64, + "current_target_version": 3, + "historical_schema_inventory_sha256": "a" * 64, + "historical_target_version": 2, + "historical_train_id": "train:source:v2", + "observed_live_version": 3, + "tier": "source", + } + + plain = cli_runner.invoke( + cli, + ["--plain", "ops", "maintenance", "migrate-tier", "source"], + catch_exceptions=False, + ) + assert plain.exit_code == 0, plain.output + assert "historical train train:source:v2 is admitted at live schema v3" in plain.output + + def test_migrate_tier_cli_refuses_live_daemon_before_sql( cli_workspace: dict[str, Path], cli_runner: CliRunner, diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 4709dd474d..01de78a01f 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1116,6 +1116,35 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( assert conn.execute("PRAGMA user_version").fetchone() == (3,) assert conn.execute("SELECT name FROM sqlite_schema WHERE name='later_items'").fetchone() == ("later_items",) assert released == [True, True] + evidence_captures = 0 + schema_inventories = 0 + canonical_inventories = 0 + real_capture = durable_change_train_module.capture_durable_database_evidence + real_schema_inventory = migration_runner.capture_durable_schema_inventory + real_canonical_inventory = durable_change_train_module._canonical_schema_inventory + + def count_evidence_captures( + connection: sqlite3.Connection, tier: ArchiveTier + ) -> migration_runner.DurableDatabaseEvidence: + nonlocal evidence_captures + evidence_captures += 1 + return real_capture(connection, tier) + + def count_schema_inventories( + connection: sqlite3.Connection, + ) -> migration_runner.DurableSchemaInventory: + nonlocal schema_inventories + schema_inventories += 1 + return real_schema_inventory(connection) + + def count_canonical_inventories(tier: ArchiveTier, target_version: int) -> migration_runner.DurableSchemaInventory: + nonlocal canonical_inventories + canonical_inventories += 1 + return real_canonical_inventory(tier, target_version) + + monkeypatch.setattr(durable_change_train_module, "capture_durable_database_evidence", count_evidence_captures) + monkeypatch.setattr(migration_runner, "capture_durable_schema_inventory", count_schema_inventories) + monkeypatch.setattr(durable_change_train_module, "_canonical_schema_inventory", count_canonical_inventories) third = execute_durable_change_train( tmp_path, ArchiveTier.SOURCE, @@ -1127,6 +1156,11 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( assert third.forward_version_receipt is not None assert third.forward_version_receipt.historical_target_version == 2 assert third.forward_version_receipt.observed_live_version == 3 + assert evidence_captures == 1 + # One live inventory plus the canonical inventory built from the fresh DDL. + # The no-op receipt reuses both instead of constructing a second pair. + assert schema_inventories == 3 + assert canonical_inventories == 1 historical_manifest = durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 2) historical_train = load_durable_change_train_manifest(historical_manifest) From 86213e4d6ee448273e3c0acea30d0a1753e97a9a Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 21:13:24 +0200 Subject: [PATCH 14/32] test(storage): use canonical evidence helper Keep the durable train cache regression test on migration_runner's declared public owner so strict mypy can resolve the symbol. --- tests/unit/storage/test_durable_change_train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 01de78a01f..f5d7a159a0 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1119,7 +1119,7 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( evidence_captures = 0 schema_inventories = 0 canonical_inventories = 0 - real_capture = durable_change_train_module.capture_durable_database_evidence + real_capture = migration_runner.capture_durable_database_evidence real_schema_inventory = migration_runner.capture_durable_schema_inventory real_canonical_inventory = durable_change_train_module._canonical_schema_inventory From 57790366ffc13eeb726a9423ba0f9c8fcbfdb2cf Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 21:37:54 +0200 Subject: [PATCH 15/32] fix(storage): bound durable admission evidence Problem: forward durable-train admission could skip an intervening unreleased schema train, and raw-failure sampling ranked the complete artifact table before applying its sample limit.\n\nWhat changed: require released evidence for every target version between the historical receipt and the live schema, and perform raw-failure sampling before correlated latest-artifact lookups. The focused tests cover both fail-closed admission and SQL shape.\n\nRef #3875\n\nCo-Authored-By: Claude --- polylogue/storage/raw_failure_lifecycle.py | 120 +++++++++++------- .../storage/sqlite/durable_change_train.py | 26 +++- tests/unit/daemon/test_raw_failure_sample.py | 6 +- .../unit/storage/test_durable_change_train.py | 13 ++ 4 files changed, 112 insertions(+), 53 deletions(-) diff --git a/polylogue/storage/raw_failure_lifecycle.py b/polylogue/storage/raw_failure_lifecycle.py index b1d733d7d3..b06fe4aad7 100644 --- a/polylogue/storage/raw_failure_lifecycle.py +++ b/polylogue/storage/raw_failure_lifecycle.py @@ -131,58 +131,82 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra is not None ) sample_limit = max(0, sample_limit) + failed_cte = """ + WITH failed AS ( + SELECT r.raw_id, r.origin, r.source_path, r.source_index, + r.validation_status, r.acquired_at_ms + FROM raw_sessions AS r + WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') + OR r.validation_status = 'failed' + ) + """ + latest_artifact_join = """ + LEFT JOIN raw_artifacts AS a + ON a.raw_id = f.raw_id + AND a.origin = f.origin + AND a.source_path = f.source_path + AND a.source_index = f.source_index + AND NOT EXISTS ( + SELECT 1 + FROM raw_artifacts AS newer + WHERE newer.raw_id = a.raw_id + AND newer.origin = a.origin + AND newer.source_path = a.source_path + AND newer.source_index = a.source_index + AND (newer.last_observed_at_ms > a.last_observed_at_ms + OR (newer.last_observed_at_ms = a.last_observed_at_ms + AND newer.artifact_id > a.artifact_id)) + ) + """ if has_artifacts: - failed_cte = """ - WITH latest_artifact AS ( - SELECT raw_id, origin, source_path, source_index, artifact_kind, support_status, - ROW_NUMBER() OVER ( - PARTITION BY raw_id, origin, source_path, source_index - ORDER BY last_observed_at_ms DESC, artifact_id DESC - ) AS artifact_rank - FROM raw_artifacts - ), failed AS ( - SELECT r.raw_id, r.origin, r.validation_status, - a.artifact_kind, a.support_status, r.acquired_at_ms - FROM raw_sessions AS r - LEFT JOIN latest_artifact AS a - ON a.raw_id = r.raw_id - AND a.origin = r.origin - AND a.source_path = r.source_path - AND a.source_index = r.source_index - AND a.artifact_rank = 1 - WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') - OR r.validation_status = 'failed' + summary_sql = ( + failed_cte + + """ + SELECT f.origin, f.validation_status, a.artifact_kind, a.support_status, + COUNT(*) AS failure_count + FROM failed AS f + """ + + latest_artifact_join + + """ + GROUP BY f.origin, f.validation_status, a.artifact_kind, a.support_status + ORDER BY f.origin, f.validation_status, a.artifact_kind, a.support_status + """ + ) + sample_sql = ( + failed_cte + + """ + , sampled AS ( + SELECT * + FROM failed + ORDER BY acquired_at_ms DESC, raw_id DESC + LIMIT ? + ) + SELECT f.raw_id, f.origin, f.validation_status, a.artifact_kind, a.support_status + FROM sampled AS f + """ + + latest_artifact_join ) - """ else: - failed_cte = """ - WITH failed AS ( - SELECT r.raw_id, r.origin, r.validation_status, - NULL AS artifact_kind, NULL AS support_status, r.acquired_at_ms - FROM raw_sessions AS r - WHERE (r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') - OR r.validation_status = 'failed' + summary_sql = ( + failed_cte + + """ + SELECT f.origin, f.validation_status, NULL, NULL, COUNT(*) AS failure_count + FROM failed AS f + GROUP BY f.origin, f.validation_status + ORDER BY f.origin, f.validation_status + """ + ) + sample_sql = ( + failed_cte + + """ + SELECT raw_id, origin, validation_status, NULL, NULL + FROM failed + ORDER BY acquired_at_ms DESC, raw_id DESC + LIMIT ? + """ ) - """ - summary_rows = conn.execute( - failed_cte - + """ - SELECT origin, validation_status, artifact_kind, support_status, COUNT(*) AS failure_count - FROM failed - GROUP BY origin, validation_status, artifact_kind, support_status - ORDER BY origin, validation_status, artifact_kind, support_status - """ - ).fetchall() - sample_rows = conn.execute( - failed_cte - + """ - SELECT raw_id, origin, validation_status, artifact_kind, support_status - FROM failed - ORDER BY acquired_at_ms DESC, raw_id DESC - LIMIT ? - """, - (sample_limit,), - ).fetchall() + summary_rows = conn.execute(summary_sql).fetchall() + sample_rows = conn.execute(sample_sql, (sample_limit,)).fetchall() except sqlite3.Error as exc: logger.warning("could not read raw failure lifecycle", exc_info=exc) return RawFailureLifecycleSnapshot(False, reason=f"could not read raw failure lifecycle: {exc}") diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 8526cb113d..478bd56913 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -1565,14 +1565,34 @@ def _forward_version_receipt_for_current_tier( ) -> DurableForwardVersionReceipt | None: """Return the newest released historical-train receipt at the live target.""" manifest_root = archive_root / ".maintenance-state" / "durable-change-trains" - historical: list[DurableChangeTrain] = [] + manifests_by_target: dict[int, DurableChangeTrain] = {} if manifest_root.is_dir(): for path in sorted(manifest_root.glob(f"{tier.value}-*.json")): train = load_durable_change_train_manifest(path) - if train.state is DurableChangeTrainState.RELEASED and train.target_version < current_version: - historical.append(train) + if train.target_version in manifests_by_target: + raise DurableChangeTrainError( + f"duplicate {tier.value} durable train manifests for target v{train.target_version}" + ) + manifests_by_target[train.target_version] = train + historical = [ + train + for train in manifests_by_target.values() + if train.state is DurableChangeTrainState.RELEASED and train.target_version < current_version + ] if not historical: return None + historical_train = max(historical, key=lambda item: item.target_version) + missing_targets = [ + version + for version in range(historical_train.target_version + 1, current_version + 1) + if manifests_by_target.get(version) is None + or manifests_by_target[version].state is not DurableChangeTrainState.RELEASED + ] + if missing_targets: + raise DurableChangeTrainError( + f"{tier.value} durable forward admission lacks released train evidence for versions " + f"{missing_targets} between v{historical_train.target_version} and live v{current_version}" + ) if evidence is None: actual = capture_durable_database_evidence(conn, tier) evidence = _DurableForwardVersionEvidence( diff --git a/tests/unit/daemon/test_raw_failure_sample.py b/tests/unit/daemon/test_raw_failure_sample.py index 4c65a2b121..90ad753e77 100644 --- a/tests/unit/daemon/test_raw_failure_sample.py +++ b/tests/unit/daemon/test_raw_failure_sample.py @@ -604,10 +604,12 @@ def open_traced_readonly(path: Path) -> sqlite3.Connection: assert snapshot.parse_failures == snapshot.unexplained == 200 assert len(snapshot.samples) == 3 - summary_queries = [statement for statement in statements if "GROUP BY origin" in statement] - sample_queries = [statement for statement in statements if "LIMIT 3" in statement] + summary_queries = [statement for statement in statements if "GROUP BY f.origin" in statement] + sample_queries = [statement for statement in statements if "FROM sampled AS f" in statement] assert len(summary_queries) == 1 assert len(sample_queries) == 1 + assert "ROW_NUMBER()" not in sample_queries[0] + assert "NOT EXISTS" in sample_queries[0] @pytest.mark.parametrize("source_state", ["missing", "malformed"]) def test_status_fails_closed_when_source_lifecycle_is_unavailable( diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index f5d7a159a0..7dc04d004b 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1116,6 +1116,19 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( assert conn.execute("PRAGMA user_version").fetchone() == (3,) assert conn.execute("SELECT name FROM sqlite_schema WHERE name='later_items'").fetchone() == ("later_items",) assert released == [True, True] + manifest_v3 = durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 3) + manifest_v3_bytes = manifest_v3.read_bytes() + manifest_v3.unlink() + with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): + execute_durable_change_train( + tmp_path, + ArchiveTier.SOURCE, + backup_manifest=None, + daemon_stopped_evidence_ref="proof:daemon-stopped", + single_writer_evidence_ref="proof:archive-ownership-lock", + release_archive_ownership=lambda: pytest.fail("missing intervening train was admitted"), + ) + manifest_v3.write_bytes(manifest_v3_bytes) evidence_captures = 0 schema_inventories = 0 canonical_inventories = 0 From 3e2bff6696423d07e2f1a0794a57e7185c296980 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 21:43:37 +0200 Subject: [PATCH 16/32] fix(storage): bind continuity checks to archive roots Problem: source continuity refresh reused a connection after its read context ended, and legacy identity compatibility lacked negative regression coverage.\n\nWhat changed: allow continuity checks to use the already-scoped archive root, pass it through source refresh validation, and pin foreign and connection-free legacy identities plus the forward target bound.\n\nRef #3875\n\nCo-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 4 ++-- polylogue/storage/sqlite/migration_runner.py | 7 +++--- .../unit/storage/test_durable_change_train.py | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 478bd56913..2943d3f129 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -870,7 +870,7 @@ def _refresh_released_source_train_continuity_locked( current, retained_current, label="source continuity retained refresh", - connection=connection, + archive_root=archive_root, ) except DurableChangeTrainError: pass @@ -915,7 +915,7 @@ def _refresh_released_source_train_continuity_locked( pre_mutation_evidence, baseline, label="source continuity pre-mutation", - connection=connection, + archive_root=archive_root, ) except DurableChangeTrainError as exc: raise DurableSourceContinuitySemanticError( diff --git a/polylogue/storage/sqlite/migration_runner.py b/polylogue/storage/sqlite/migration_runner.py index 61cf7b054e..2d15a366c7 100644 --- a/polylogue/storage/sqlite/migration_runner.py +++ b/polylogue/storage/sqlite/migration_runner.py @@ -1653,16 +1653,17 @@ def _assert_durable_database_continuity( expected: DurableDatabaseEvidence, *, label: str, + archive_root: Path | None = None, connection: sqlite3.Connection | None = None, ) -> None: """Require the live durable file to retain its authenticated evidence.""" identity_continuous = actual.archive_identity_digest == expected.archive_identity_digest - if not identity_continuous and connection is not None: - archive_root = _connection_main_path(connection).parent + if not identity_continuous and (archive_root is not None or connection is not None): + resolved_archive_root = archive_root or _connection_main_path(cast(sqlite3.Connection, connection)).parent identity_continuous = _archive_identity_continuity_matches( actual.archive_identity_digest, expected.archive_identity_digest, - archive_root, + resolved_archive_root, ) if ( actual.quick_check != expected.quick_check diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 7dc04d004b..9f0da9c29b 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1194,6 +1194,15 @@ def count_canonical_inventories(tier: ArchiveTier, target_version: int) -> migra assert ( receipt.historical_schema_inventory_sha256 == historical_train.proof.fresh_ddl_parity.migrated_inventory_sha256 ) + with sqlite3.connect(db_path) as conn: + with pytest.raises(DurableChangeTrainError, match="is newer than current target"): + durable_change_train_module._verify_released_train_live_tier( + tmp_path, + conn, + historical_train, + current_target_version=2, + actual_evidence=actual, + ) captures = 0 real_capture = migration_runner.capture_durable_database_evidence @@ -1252,6 +1261,19 @@ def test_continuity_admits_legacy_full_archive_identity_digest(tmp_path: Path) - label="legacy identity compatibility", connection=conn, ) + with pytest.raises(DurableChangeTrainError, match="continuity proof failed"): + migration_runner._assert_durable_database_continuity( + current, + replace(current, archive_identity_digest="a" * 64), + label="foreign identity", + connection=conn, + ) + with pytest.raises(DurableChangeTrainError, match="continuity proof failed"): + migration_runner._assert_durable_database_continuity( + current, + legacy, + label="legacy identity without connection", + ) def test_future_train_sidecar_hash_and_slot_are_admission_bound( From 7ce12cb1c816d596500029b93974e9c1b0403227 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 22:03:54 +0200 Subject: [PATCH 17/32] fix(storage): scope durable train identities Problem: a source train was bound to both durable tier identities, so creating a previously absent user tier could invalidate source continuity. Legacy identity matching also depended on an unresolved archive-root spelling.\n\nWhat changed: bind new evidence to the migrated tier, normalize legacy roots, and require historical schema evidence for every intervening released train. Add a regression for late user-tier initialization.\n\nRef #3875\n\nCo-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 12 +++++++++++- polylogue/storage/sqlite/migration_runner.py | 17 ++++++++++------- tests/unit/storage/test_durable_change_train.py | 15 +++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 2943d3f129..35b1e32982 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -925,6 +925,7 @@ def _refresh_released_source_train_continuity_locked( pre_mutation_evidence.archive_identity_digest, train.apply_evidence.post.archive_identity_digest, archive_root, + ArchiveTier.SOURCE, ): raise DurableSourceContinuitySemanticError( "source continuity refresh pre-state has the wrong archive identity" @@ -933,6 +934,7 @@ def _refresh_released_source_train_continuity_locked( current.archive_identity_digest, train.apply_evidence.post.archive_identity_digest, archive_root, + ArchiveTier.SOURCE, ): raise DurableSourceContinuitySemanticError("source continuity refresh changed archive identity") if pre_mutation_evidence.quick_check != ("ok",) or current.quick_check != ("ok",): @@ -1499,7 +1501,12 @@ def _verify_released_train_live_tier( return None historical = _historical_schema_evidence(train) expected_identity = train.apply_evidence.post.archive_identity_digest - if not _archive_identity_continuity_matches(actual.archive_identity_digest, expected_identity, archive_root): + if not _archive_identity_continuity_matches( + actual.archive_identity_digest, + expected_identity, + archive_root, + train.tier, + ): raise DurableChangeTrainError( f"{train.tier.value} durable tier immutable archive identity differs from historical train " f"v{train.target_version} after later train advancement" @@ -1593,6 +1600,9 @@ def _forward_version_receipt_for_current_tier( f"{tier.value} durable forward admission lacks released train evidence for versions " f"{missing_targets} between v{historical_train.target_version} and live v{current_version}" ) + for version in range(historical_train.target_version + 1, current_version + 1): + intervening = manifests_by_target[version] + _historical_schema_evidence(intervening) if evidence is None: actual = capture_durable_database_evidence(conn, tier) evidence = _DurableForwardVersionEvidence( diff --git a/polylogue/storage/sqlite/migration_runner.py b/polylogue/storage/sqlite/migration_runner.py index 2d15a366c7..05c89a9396 100644 --- a/polylogue/storage/sqlite/migration_runner.py +++ b/polylogue/storage/sqlite/migration_runner.py @@ -1608,11 +1608,11 @@ def capture_durable_database_evidence( from polylogue.storage.archive_identity import ArchiveIdentity # Durable migration evidence must survive replacement of rebuildable - # generations. The source/user tier identities are the durable authority; - # active index, embeddings, and ops identities belong to derived/runtime - # state and must not invalidate a durable train. - durable_id = ArchiveIdentity.resolve(live_path.parent).durable_id - archive_identity_digest = hashlib.sha256(durable_id.encode("utf-8")).hexdigest() + # generations and creation of another durable tier. Bind a train to the + # file it actually migrates, not to the whole durable pair: a source train + # must remain valid when a previously absent user.db is initialized later. + tier_identity = ArchiveIdentity.resolve(live_path.parent).tier(tier.value).stable_id + archive_identity_digest = hashlib.sha256(tier_identity.encode("utf-8")).hexdigest() content_hasher = hashlib.sha256() for statement in conn.iterdump(): content_hasher.update(statement.encode("utf-8")) @@ -1633,14 +1633,16 @@ def _archive_identity_continuity_matches( actual_digest: str, expected_digest: str, archive_root: Path, + tier: ArchiveTier, ) -> bool: if actual_digest == expected_digest: return True from polylogue.storage.archive_identity import ArchiveIdentity - identity = ArchiveIdentity.resolve(archive_root) + identity = ArchiveIdentity.resolve(archive_root.resolve()) legacy_digest = identity.authority_identity_digest - durable_digest = hashlib.sha256(identity.durable_id.encode("utf-8")).hexdigest() + tier_identity = identity.tier(tier.value).stable_id + durable_digest = hashlib.sha256(tier_identity.encode("utf-8")).hexdigest() # Manifests written before the durable-tier identity split contain the # old full-archive digest. Admit that legacy evidence only when the # current archive still has the same legacy identity and the newly @@ -1664,6 +1666,7 @@ def _assert_durable_database_continuity( actual.archive_identity_digest, expected.archive_identity_digest, resolved_archive_root, + actual.tier, ) if ( actual.quick_check != expected.quick_check diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 9f0da9c29b..f35f2a32e6 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1276,6 +1276,21 @@ def test_continuity_admits_legacy_full_archive_identity_digest(tmp_path: Path) - ) +def test_source_train_identity_survives_late_user_tier_initialization(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database + + source_path = tmp_path / "source.db" + initialize_archive_database(source_path, ArchiveTier.SOURCE) + with sqlite3.connect(source_path) as conn: + before = migration_runner.capture_durable_database_evidence(conn, ArchiveTier.SOURCE) + + initialize_archive_database(tmp_path / "user.db", ArchiveTier.USER) + with sqlite3.connect(source_path) as conn: + after = migration_runner.capture_durable_database_evidence(conn, ArchiveTier.SOURCE) + + assert after.archive_identity_digest == before.archive_identity_digest + + def test_future_train_sidecar_hash_and_slot_are_admission_bound( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From d3c994c44f0baeb894a1d67a9b7923645004ce51 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 22:15:53 +0200 Subject: [PATCH 18/32] fix(storage): enforce train chains during startup Problem: startup reconciliation validated each released durable train independently, so a missing intervening manifest could bypass the complete-chain admission guard that maintenance already enforced. What changed: share the released-train chain loader and schema-proof check between maintenance admission and startup reconciliation. Add a regression that removes the current manifest and proves startup refuses the incomplete chain. Compatibility/migration: archives with complete released train evidence retain the existing path; incomplete durable-train histories now fail closed. Ref polylogue-dcrmm Co-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 77 +++++++++++++------ .../unit/storage/test_durable_change_train.py | 3 + 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 35b1e32982..cda13aa588 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -1572,15 +1572,7 @@ def _forward_version_receipt_for_current_tier( ) -> DurableForwardVersionReceipt | None: """Return the newest released historical-train receipt at the live target.""" manifest_root = archive_root / ".maintenance-state" / "durable-change-trains" - manifests_by_target: dict[int, DurableChangeTrain] = {} - if manifest_root.is_dir(): - for path in sorted(manifest_root.glob(f"{tier.value}-*.json")): - train = load_durable_change_train_manifest(path) - if train.target_version in manifests_by_target: - raise DurableChangeTrainError( - f"duplicate {tier.value} durable train manifests for target v{train.target_version}" - ) - manifests_by_target[train.target_version] = train + manifests_by_target = _released_train_manifests_by_target(manifest_root, tier) historical = [ train for train in manifests_by_target.values() @@ -1589,20 +1581,12 @@ def _forward_version_receipt_for_current_tier( if not historical: return None historical_train = max(historical, key=lambda item: item.target_version) - missing_targets = [ - version - for version in range(historical_train.target_version + 1, current_version + 1) - if manifests_by_target.get(version) is None - or manifests_by_target[version].state is not DurableChangeTrainState.RELEASED - ] - if missing_targets: - raise DurableChangeTrainError( - f"{tier.value} durable forward admission lacks released train evidence for versions " - f"{missing_targets} between v{historical_train.target_version} and live v{current_version}" - ) - for version in range(historical_train.target_version + 1, current_version + 1): - intervening = manifests_by_target[version] - _historical_schema_evidence(intervening) + _require_released_train_chain( + tier, + manifests_by_target, + historical_target_version=historical_train.target_version, + current_version=current_version, + ) if evidence is None: actual = capture_durable_database_evidence(conn, tier) evidence = _DurableForwardVersionEvidence( @@ -1627,6 +1611,47 @@ def _forward_version_receipt_for_current_tier( return None +def _released_train_manifests_by_target( + manifest_root: Path, + tier: ArchiveTier, +) -> dict[int, DurableChangeTrain]: + """Load one persisted train record per target version for a durable tier.""" + manifests_by_target: dict[int, DurableChangeTrain] = {} + if not manifest_root.is_dir(): + return manifests_by_target + for path in sorted(manifest_root.glob(f"{tier.value}-*.json")): + train = load_durable_change_train_manifest(path) + if train.target_version in manifests_by_target: + raise DurableChangeTrainError( + f"duplicate {tier.value} durable train manifests for target v{train.target_version}" + ) + manifests_by_target[train.target_version] = train + return manifests_by_target + + +def _require_released_train_chain( + tier: ArchiveTier, + manifests_by_target: dict[int, DurableChangeTrain], + *, + historical_target_version: int, + current_version: int, +) -> None: + """Require released, schema-proven evidence for every later version.""" + missing_targets = [ + version + for version in range(historical_target_version + 1, current_version + 1) + if manifests_by_target.get(version) is None + or manifests_by_target[version].state is not DurableChangeTrainState.RELEASED + ] + if missing_targets: + raise DurableChangeTrainError( + f"{tier.value} durable forward admission lacks released train evidence for versions " + f"{missing_targets} between v{historical_target_version} and live v{current_version}" + ) + for version in range(historical_target_version + 1, current_version + 1): + _historical_schema_evidence(manifests_by_target[version]) + + def _prove_and_release_persisted_train( archive_root: Path, manifest_path: Path, @@ -1931,6 +1956,12 @@ def _reconcile_durable_change_train_startup_locked( actual = capture_durable_database_evidence(live, train.tier) live_evidence_by_tier[train.tier] = actual if actual.user_version > train.target_version: + _require_released_train_chain( + train.tier, + _released_train_manifests_by_target(manifest_root, train.tier), + historical_target_version=train.target_version, + current_version=actual.user_version, + ) if train.tier not in live_integrity_by_tier: live_integrity_by_tier[train.tier] = tuple( str(row[0]) for row in live.execute("PRAGMA integrity_check") diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index f35f2a32e6..37abd81615 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1119,6 +1119,8 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( manifest_v3 = durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 3) manifest_v3_bytes = manifest_v3.read_bytes() manifest_v3.unlink() + with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): + durable_change_train_module.reconcile_durable_change_train_startup(tmp_path) with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): execute_durable_change_train( tmp_path, @@ -1225,6 +1227,7 @@ def count_captures(connection: sqlite3.Connection, tier: ArchiveTier) -> migrati unrelated_manifest = durable_change_train_manifest_path(unrelated_root, ArchiveTier.SOURCE, 2) unrelated_manifest.parent.mkdir(parents=True) shutil.copy2(historical_manifest, unrelated_manifest) + shutil.copy2(manifest_v3, durable_change_train_manifest_path(unrelated_root, ArchiveTier.SOURCE, 3)) with sqlite3.connect(unrelated_root / "source.db") as conn: assert conn.execute("PRAGMA integrity_check").fetchone() == ("ok",) with pytest.raises(DurableChangeTrainError, match="immutable archive identity differs"): From 857bd1cc267a2e3f3310e1f6b09387e8691104e4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 22:25:06 +0200 Subject: [PATCH 19/32] test(storage): pin startup chain rejection detail What changed: retain the historical manifest path needed by the later receipt assertions and pin startup rejection to the missing current target version.\n\nVerification: tests/unit/storage/test_durable_change_train.py passed 44 tests.\n\nRef polylogue-dcrmm\n\nCo-Authored-By: Claude --- tests/unit/storage/test_durable_change_train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 37abd81615..c49109e2b9 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1116,10 +1116,11 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( assert conn.execute("PRAGMA user_version").fetchone() == (3,) assert conn.execute("SELECT name FROM sqlite_schema WHERE name='later_items'").fetchone() == ("later_items",) assert released == [True, True] + historical_manifest = durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 2) manifest_v3 = durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 3) manifest_v3_bytes = manifest_v3.read_bytes() manifest_v3.unlink() - with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): + with pytest.raises(DurableChangeTrainError, match=r"versions \[3\]"): durable_change_train_module.reconcile_durable_change_train_startup(tmp_path) with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): execute_durable_change_train( @@ -1177,7 +1178,6 @@ def count_canonical_inventories(tier: ArchiveTier, target_version: int) -> migra assert schema_inventories == 3 assert canonical_inventories == 1 - historical_manifest = durable_change_train_manifest_path(tmp_path, ArchiveTier.SOURCE, 2) historical_train = load_durable_change_train_manifest(historical_manifest) with sqlite3.connect(db_path) as conn: actual = migration_runner.capture_durable_database_evidence(conn, ArchiveTier.SOURCE) From 48f005b4d907a189f217c1b52730ced384e34dd3 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 22:28:01 +0200 Subject: [PATCH 20/32] test(storage): assert startup rejection is non-mutating What changed: assert that startup refusal of an incomplete released-train chain does not release or otherwise mutate the durable-train ownership result. Verification: tests/unit/storage/test_durable_change_train.py passed 44 tests. Ref polylogue-dcrmm Co-Authored-By: Claude --- tests/unit/storage/test_durable_change_train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index c49109e2b9..3cfd7d1001 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1122,6 +1122,7 @@ def test_maintenance_route_replays_historical_sidecars_before_current_target( manifest_v3.unlink() with pytest.raises(DurableChangeTrainError, match=r"versions \[3\]"): durable_change_train_module.reconcile_durable_change_train_startup(tmp_path) + assert released == [True, True] with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): execute_durable_change_train( tmp_path, From 079f434cb2ea3724e7a0d0431c41e42bbb0683a2 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 22:43:38 +0200 Subject: [PATCH 21/32] fix(storage): close durable train admission gaps Problem: forward admission accepted a chain that skipped the adoption floor, startup validated released trains before recovering later committed trains, and legacy source continuity refreshes could fail manifest validation after the durable identity split. What changed: normalize retained legacy post-apply identity evidence during an authenticated continuity refresh, require every released train after the adoption floor, and recover all non-released lifecycle states before checking released manifests. Focused tests cover each ordering and compatibility boundary. Verification: tests/unit/storage/test_durable_change_train.py passed 46 tests; devtools verify --quick passed all 24 steps. Ref polylogue-dcrmm Co-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 123 +++++++++++------- .../unit/storage/test_durable_change_train.py | 112 +++++++++++++++- 2 files changed, 185 insertions(+), 50 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index cda13aa588..cb0f8f1e15 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -996,9 +996,19 @@ def _refresh_released_source_train_continuity_locked( references = _migration_runner._append_proof_refs(train.proof_refs, evidence_ref, refresh_ref) if train.proof is None: raise DurableChangeTrainError("source continuity refresh requires train proof") + retained_apply_evidence = train.apply_evidence + if current.archive_identity_digest != retained_apply_evidence.post.archive_identity_digest: + retained_apply_evidence = replace( + retained_apply_evidence, + post=replace( + retained_apply_evidence.post, + archive_identity_digest=current.archive_identity_digest, + ), + ) updated = replace( train, revision=train.revision + 1, + apply_evidence=retained_apply_evidence, source_continuity_evidence=current, proof_refs=references, ) @@ -1639,16 +1649,17 @@ def _require_released_train_chain( """Require released, schema-proven evidence for every later version.""" missing_targets = [ version - for version in range(historical_target_version + 1, current_version + 1) + for version in range(DURABLE_MIGRATION_ADOPTION_FLOORS[tier] + 1, current_version + 1) if manifests_by_target.get(version) is None or manifests_by_target[version].state is not DurableChangeTrainState.RELEASED ] if missing_targets: raise DurableChangeTrainError( f"{tier.value} durable forward admission lacks released train evidence for versions " - f"{missing_targets} between v{historical_target_version} and live v{current_version}" + f"{missing_targets} from adoption floor v{DURABLE_MIGRATION_ADOPTION_FLOORS[tier]} " + f"through live v{current_version}" ) - for version in range(historical_target_version + 1, current_version + 1): + for version in range(DURABLE_MIGRATION_ADOPTION_FLOORS[tier] + 1, current_version + 1): _historical_schema_evidence(manifests_by_target[version]) @@ -1920,7 +1931,18 @@ def _reconcile_durable_change_train_startup_locked( live_integrity_by_tier: dict[ArchiveTier, tuple[str, ...]] = {} live_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} canonical_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} - for manifest_path in sorted(manifest_root.glob("*.json")): + manifest_paths = tuple(sorted(manifest_root.glob("*.json"))) + + def record_reconciled(path: Path) -> None: + if path not in reconciled: + reconciled.append(path) + + # Recover every non-released lifecycle state before validating any + # released train. A later committed train may be persisted as + # backup-authorized, applied, or proven while an older released manifest + # is still present. Checking the older train first would reject the + # incomplete chain before startup had a chance to finish that recovery. + for manifest_path in manifest_paths: train = load_durable_change_train_manifest(manifest_path) if train.state is DurableChangeTrainState.FAILED: tier_path = archive_root / f"{train.tier.value}.db" @@ -1932,7 +1954,7 @@ def _reconcile_durable_change_train_startup_locked( writer_release_evidence_ref=f"proof:startup-writer-release:{train.train_id}", ) train = _persist_train_transition(manifest_path, recovered, expected_revision=train.revision) - reconciled.append(manifest_path) + record_reconciled(manifest_path) if train.state is DurableChangeTrainState.ADMITTED: continue if train.state is DurableChangeTrainState.BACKUP_AUTHORIZED: @@ -1949,54 +1971,57 @@ def _reconcile_durable_change_train_startup_locked( _persist_train_transition(manifest_path, exc.failed_train, expected_revision=train.revision) raise train = _persist_train_transition(manifest_path, recovered, expected_revision=train.revision) - if train.state is DurableChangeTrainState.RELEASED: - with _open_existing_tier(archive_root / f"{train.tier.value}.db") as live: - actual = live_evidence_by_tier.get(train.tier) - if actual is None: - actual = capture_durable_database_evidence(live, train.tier) - live_evidence_by_tier[train.tier] = actual - if actual.user_version > train.target_version: - _require_released_train_chain( - train.tier, - _released_train_manifests_by_target(manifest_root, train.tier), - historical_target_version=train.target_version, - current_version=actual.user_version, - ) - if train.tier not in live_integrity_by_tier: - live_integrity_by_tier[train.tier] = tuple( - str(row[0]) for row in live.execute("PRAGMA integrity_check") - ) - if train.tier not in live_inventory_by_tier: - live_inventory_by_tier[train.tier] = _migration_runner.capture_durable_schema_inventory(live) - if train.tier not in canonical_inventory_by_tier: - canonical_inventory_by_tier[train.tier] = _canonical_schema_inventory( - train.tier, actual.user_version - ) - if live_evidence_cache is not None: - live_evidence_cache[train.tier] = _DurableForwardVersionEvidence( - actual=actual, - integrity_check=live_integrity_by_tier[train.tier], - live_inventory=live_inventory_by_tier[train.tier], - canonical_inventory=canonical_inventory_by_tier[train.tier], - ) - _verify_released_train_live_tier( - archive_root, - live, - train, - actual_evidence=actual, - integrity_check=live_integrity_by_tier.get(train.tier), - live_inventory=live_inventory_by_tier.get(train.tier), - canonical_inventory=canonical_inventory_by_tier.get(train.tier), - ) - reconciled.append(manifest_path) - continue - if train.state not in { + + if train.state in { DurableChangeTrainState.APPLIED, DurableChangeTrainState.PROVEN, }: + _prove_and_release_persisted_train(archive_root, manifest_path, train) + record_reconciled(manifest_path) + + for manifest_path in manifest_paths: + train = load_durable_change_train_manifest(manifest_path) + if train.state is not DurableChangeTrainState.RELEASED: continue - _prove_and_release_persisted_train(archive_root, manifest_path, train) - reconciled.append(manifest_path) + with _open_existing_tier(archive_root / f"{train.tier.value}.db") as live: + actual = live_evidence_by_tier.get(train.tier) + if actual is None: + actual = capture_durable_database_evidence(live, train.tier) + live_evidence_by_tier[train.tier] = actual + if actual.user_version > train.target_version: + _require_released_train_chain( + train.tier, + _released_train_manifests_by_target(manifest_root, train.tier), + historical_target_version=train.target_version, + current_version=actual.user_version, + ) + if train.tier not in live_integrity_by_tier: + live_integrity_by_tier[train.tier] = tuple( + str(row[0]) for row in live.execute("PRAGMA integrity_check") + ) + if train.tier not in live_inventory_by_tier: + live_inventory_by_tier[train.tier] = _migration_runner.capture_durable_schema_inventory(live) + if train.tier not in canonical_inventory_by_tier: + canonical_inventory_by_tier[train.tier] = _canonical_schema_inventory( + train.tier, actual.user_version + ) + if live_evidence_cache is not None: + live_evidence_cache[train.tier] = _DurableForwardVersionEvidence( + actual=actual, + integrity_check=live_integrity_by_tier[train.tier], + live_inventory=live_inventory_by_tier[train.tier], + canonical_inventory=canonical_inventory_by_tier[train.tier], + ) + _verify_released_train_live_tier( + archive_root, + live, + train, + actual_evidence=actual, + integrity_check=live_integrity_by_tier.get(train.tier), + live_inventory=live_inventory_by_tier.get(train.tier), + canonical_inventory=canonical_inventory_by_tier.get(train.tier), + ) + record_reconciled(manifest_path) return tuple(reconciled) diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 3cfd7d1001..0d51c73ec7 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -377,6 +377,8 @@ def test_released_source_train_can_record_an_authorized_mutation_refresh( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: + from polylogue.storage.archive_identity import ArchiveIdentity + db_path = tmp_path / "source.db" _create_current_database(db_path) _install_synthetic_migration(tmp_path, monkeypatch, ArchiveTier.SOURCE) @@ -400,6 +402,17 @@ def test_released_source_train_can_record_an_authorized_mutation_refresh( restart_convergence=restart, ) train = release_durable_change_train(train, evidence_ref="proof:release") + assert train.apply_evidence is not None + train = replace( + train, + apply_evidence=replace( + train.apply_evidence, + post=replace( + train.apply_evidence.post, + archive_identity_digest=ArchiveIdentity.resolve(tmp_path).authority_identity_digest, + ), + ), + ) manifest = tmp_path / ".maintenance-state" / "durable-change-trains" / "source-002.json" manifest.parent.mkdir(parents=True) write_durable_change_train_manifest(manifest, train, expected_revision=-1) @@ -458,7 +471,10 @@ def record_refresh_fsync(path: Path) -> None: assert refreshed.state is DurableChangeTrainState.RELEASED assert refreshed.source_continuity_evidence is not None assert released.apply_evidence is not None - assert refreshed.apply_evidence == released.apply_evidence + assert refreshed.apply_evidence is not None + assert released.apply_evidence is not None + assert refreshed.apply_evidence.pre == released.apply_evidence.pre + assert refreshed.apply_evidence.post.archive_identity_digest != released.apply_evidence.post.archive_identity_digest assert refreshed.proof == released.proof assert refreshed.revision == released.revision + 1 assert any(ref.startswith("proof:source-continuity-refresh:") for ref in refreshed.proof_refs) @@ -1280,6 +1296,100 @@ def test_continuity_admits_legacy_full_archive_identity_digest(tmp_path: Path) - ) +def test_released_train_chain_is_anchored_at_adoption_floor() -> None: + floor = DURABLE_MIGRATION_ADOPTION_FLOORS[ArchiveTier.SOURCE] + released = cast(DurableChangeTrain, SimpleNamespace(state=DurableChangeTrainState.RELEASED)) + + with pytest.raises(DurableChangeTrainError, match=rf"versions \[{floor + 1}\]"): + durable_change_train_module._require_released_train_chain( + ArchiveTier.SOURCE, + { + floor + 2: released, + floor + 3: released, + }, + historical_target_version=floor + 2, + current_version=floor + 3, + ) + + +def test_startup_recovers_later_train_before_released_chain_validation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + manifest_root = tmp_path / ".maintenance-state" / "durable-change-trains" + manifest_root.mkdir(parents=True) + first_path = manifest_root / "source-027.json" + later_path = manifest_root / "source-028.json" + first_path.touch() + later_path.touch() + released = cast( + DurableChangeTrain, + SimpleNamespace(state=DurableChangeTrainState.RELEASED, tier=ArchiveTier.SOURCE, target_version=27), + ) + later_released = cast( + DurableChangeTrain, + SimpleNamespace(state=DurableChangeTrainState.RELEASED, tier=ArchiveTier.SOURCE, target_version=28), + ) + backup_authorized = cast( + DurableChangeTrain, + SimpleNamespace( + state=DurableChangeTrainState.BACKUP_AUTHORIZED, + tier=ArchiveTier.SOURCE, + target_version=28, + train_id="train:source:v28", + revision=0, + ), + ) + states = {first_path: released, later_path: backup_authorized} + events: list[tuple[str, int]] = [] + + @contextmanager + def fake_open_tier(_path: Path) -> Iterator[sqlite3.Connection]: + with sqlite3.connect(":memory:") as connection: + yield connection + + def fake_load(path: Path) -> DurableChangeTrain: + return states[path] + + def fake_persist(path: Path, train: DurableChangeTrain, *, expected_revision: int) -> DurableChangeTrain: + states[path] = train + return train + + def fake_recover(*_args: object, **_kwargs: object) -> DurableChangeTrain: + events.append(("recover", 28)) + return later_released + + def fake_capture(*_args: object, **_kwargs: object) -> SimpleNamespace: + return SimpleNamespace(user_version=28) + + monkeypatch.setattr(durable_change_train_module, "_recover_pending_source_continuity_intents", lambda _root: None) + monkeypatch.setattr(durable_change_train_module, "_open_existing_tier", fake_open_tier) + monkeypatch.setattr(durable_change_train_module, "load_durable_change_train_manifest", fake_load) + monkeypatch.setattr(durable_change_train_module, "_persist_train_transition", fake_persist) + monkeypatch.setattr(durable_change_train_module, "reconcile_interrupted_durable_change_train", fake_recover) + monkeypatch.setattr(durable_change_train_module, "capture_durable_database_evidence", fake_capture) + monkeypatch.setattr(durable_change_train_module, "_historical_schema_evidence", lambda _train: None) + monkeypatch.setattr( + migration_runner, + "capture_durable_schema_inventory", + lambda _connection: SimpleNamespace(sha256="inventory"), + ) + monkeypatch.setattr( + durable_change_train_module, + "_canonical_schema_inventory", + lambda _tier, _version: SimpleNamespace(sha256="canonical"), + ) + monkeypatch.setattr( + durable_change_train_module, + "_verify_released_train_live_tier", + lambda _root, _connection, train, **_kwargs: events.append(("verify", train.target_version)), + ) + + durable_change_train_module._reconcile_durable_change_train_startup_locked(tmp_path) + + assert events == [("recover", 28), ("verify", 27), ("verify", 28)] + + def test_source_train_identity_survives_late_user_tier_initialization(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database From 927c958661e94e3a222a601d13daacc98ea63ad1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 22:52:09 +0200 Subject: [PATCH 22/32] refactor(storage): remove stale train-chain argument Problem: the adoption-floor chain validator retained a historical target parameter that no longer influenced validation after the chain rule was corrected.\n\nWhat changed: remove the unused parameter and its now-unneeded local selection, then update the focused regression call site.\n\nVerification: devtools test tests/unit/storage/test_durable_change_train.py (46 passed).\n\nCo-Authored-By: Claude --- polylogue/storage/sqlite/durable_change_train.py | 4 ---- tests/unit/storage/test_durable_change_train.py | 1 - 2 files changed, 5 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index cb0f8f1e15..fa6870d050 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -1590,11 +1590,9 @@ def _forward_version_receipt_for_current_tier( ] if not historical: return None - historical_train = max(historical, key=lambda item: item.target_version) _require_released_train_chain( tier, manifests_by_target, - historical_target_version=historical_train.target_version, current_version=current_version, ) if evidence is None: @@ -1643,7 +1641,6 @@ def _require_released_train_chain( tier: ArchiveTier, manifests_by_target: dict[int, DurableChangeTrain], *, - historical_target_version: int, current_version: int, ) -> None: """Require released, schema-proven evidence for every later version.""" @@ -1992,7 +1989,6 @@ def record_reconciled(path: Path) -> None: _require_released_train_chain( train.tier, _released_train_manifests_by_target(manifest_root, train.tier), - historical_target_version=train.target_version, current_version=actual.user_version, ) if train.tier not in live_integrity_by_tier: diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 0d51c73ec7..87fc82dd8c 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1307,7 +1307,6 @@ def test_released_train_chain_is_anchored_at_adoption_floor() -> None: floor + 2: released, floor + 3: released, }, - historical_target_version=floor + 2, current_version=floor + 3, ) From ba754f5b3c752aa0476546e69fdaffc3169bb899 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 23:09:12 +0200 Subject: [PATCH 23/32] fix(storage): close durable train review gaps Problem: automated review identified stale evidence boundaries, an incomplete historical-chain check, recovery reporting loss, and raw-failure samples that did not prioritize unexplained rows or cover the no-artifact route.\n\nWhat changed: bind receipt serialization through one local, constrain legacy identity rewrites, validate the adoption-floor chain before empty-history returns, cache manifest maps, retain recovery accounting, document lease ownership, prioritize unexplained samples in SQL, and add focused regression coverage.\n\nAlternatives rejected: a new raw_artifacts source index would require a numbered durable migration and target-version change, so it remains outside this admission hardening change.\n\nVerification: devtools test tests/unit/daemon/test_raw_failure_sample.py tests/unit/storage/test_durable_change_train.py tests/unit/cli/test_archive_maintenance_cli.py::test_migrate_tier_cli_exposes_forward_version_receipt (84 passed). The two rebuild-index tests remain pre-existing failures in the mmap-budget fixture path.\n\nCo-Authored-By: Claude --- .../cli/commands/maintenance/_migrate_tier.py | 20 ++++----- polylogue/storage/raw_failure_lifecycle.py | 24 +++++++--- .../storage/sqlite/durable_change_train.py | 36 ++++++++++----- .../unit/cli/test_archive_maintenance_cli.py | 1 + tests/unit/daemon/test_raw_failure_sample.py | 44 ++++++++++++++++++- .../unit/storage/test_durable_change_train.py | 41 ++++++++++++++++- 6 files changed, 137 insertions(+), 29 deletions(-) diff --git a/polylogue/cli/commands/maintenance/_migrate_tier.py b/polylogue/cli/commands/maintenance/_migrate_tier.py index 8efa19219b..87f47a4acf 100644 --- a/polylogue/cli/commands/maintenance/_migrate_tier.py +++ b/polylogue/cli/commands/maintenance/_migrate_tier.py @@ -121,6 +121,7 @@ def migrate_tier_command( raise SystemExit(1) from exc result = execution.migration_result if execution is not None else None + receipt = execution.forward_version_receipt if execution is not None else None payload = { "ok": True, "tier": tier, @@ -140,15 +141,15 @@ def migrate_tier_command( "applied_versions": list(result.applied_versions) if result is not None else [], "forward_version_receipt": ( { - "tier": execution.forward_version_receipt.tier.value, - "historical_train_id": execution.forward_version_receipt.historical_train_id, - "historical_target_version": execution.forward_version_receipt.historical_target_version, - "current_target_version": execution.forward_version_receipt.current_target_version, - "observed_live_version": execution.forward_version_receipt.observed_live_version, - "historical_schema_inventory_sha256": execution.forward_version_receipt.historical_schema_inventory_sha256, - "archive_identity_digest": execution.forward_version_receipt.archive_identity_digest, + "tier": receipt.tier.value, + "historical_train_id": receipt.historical_train_id, + "historical_target_version": receipt.historical_target_version, + "current_target_version": receipt.current_target_version, + "observed_live_version": receipt.observed_live_version, + "historical_schema_inventory_sha256": receipt.historical_schema_inventory_sha256, + "archive_identity_digest": receipt.archive_identity_digest, } - if execution is not None and execution.forward_version_receipt is not None + if receipt is not None else None ), } @@ -160,8 +161,7 @@ def migrate_tier_command( click.echo(f"Initialized missing {tier} tier at schema version {initialized_version}.") return if result is None: - if execution is not None and execution.forward_version_receipt is not None: - receipt = execution.forward_version_receipt + if receipt is not None: click.echo( f"No pending durable migration for {tier}; historical train {receipt.historical_train_id} " f"is admitted at live schema v{receipt.observed_live_version} " diff --git a/polylogue/storage/raw_failure_lifecycle.py b/polylogue/storage/raw_failure_lifecycle.py index b06fe4aad7..21a3ad1ff0 100644 --- a/polylogue/storage/raw_failure_lifecycle.py +++ b/polylogue/storage/raw_failure_lifecycle.py @@ -176,15 +176,27 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra failed_cte + """ , sampled AS ( - SELECT * - FROM failed - ORDER BY acquired_at_ms DESC, raw_id DESC + SELECT f.raw_id, f.origin, f.validation_status, f.acquired_at_ms, + a.artifact_kind, a.support_status + FROM failed AS f + """ + + latest_artifact_join + + """ + ORDER BY CASE + WHEN f.validation_status = 'failed' THEN 0 + WHEN (a.artifact_kind, a.support_status) IN ( + ('deferred_hot_jsonl_capture', 'partial_decode'), + ('terminal_corrupt_input', 'decode_failed'), + ('terminal_unsupported_shape', 'unsupported_parseable') + ) THEN 1 + ELSE 0 + END, + f.acquired_at_ms DESC, f.raw_id DESC LIMIT ? ) - SELECT f.raw_id, f.origin, f.validation_status, a.artifact_kind, a.support_status - FROM sampled AS f + SELECT raw_id, origin, validation_status, artifact_kind, support_status + FROM sampled """ - + latest_artifact_join ) else: summary_sql = ( diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index fa6870d050..e28b360b4b 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -794,7 +794,7 @@ def _refresh_released_source_train_continuity_locked( backup bytes to a separate current-evidence record. The original migration evidence remains immutable in ``apply_evidence``. """ - from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation + from polylogue.storage.archive_identity import ArchiveIdentity, ArchiveLocation, OwnedArchiveLocation archive_root = archive_root.resolve() mutation_receipt = mutation_receipt.resolve() @@ -997,7 +997,11 @@ def _refresh_released_source_train_continuity_locked( if train.proof is None: raise DurableChangeTrainError("source continuity refresh requires train proof") retained_apply_evidence = train.apply_evidence - if current.archive_identity_digest != retained_apply_evidence.post.archive_identity_digest: + legacy_archive_identity_digest = ArchiveIdentity.resolve(archive_root).authority_identity_digest + if ( + retained_apply_evidence.post.archive_identity_digest == legacy_archive_identity_digest + and current.archive_identity_digest != retained_apply_evidence.post.archive_identity_digest + ): retained_apply_evidence = replace( retained_apply_evidence, post=replace( @@ -1588,13 +1592,14 @@ def _forward_version_receipt_for_current_tier( for train in manifests_by_target.values() if train.state is DurableChangeTrainState.RELEASED and train.target_version < current_version ] + if current_version > DURABLE_MIGRATION_ADOPTION_FLOORS[tier]: + _require_released_train_chain( + tier, + manifests_by_target, + current_version=current_version, + ) if not historical: return None - _require_released_train_chain( - tier, - manifests_by_target, - current_version=current_version, - ) if evidence is None: actual = capture_durable_database_evidence(conn, tier) evidence = _DurableForwardVersionEvidence( @@ -1726,7 +1731,11 @@ def execute_durable_change_train( runtime_consumer_results: Sequence[DurableRuntimeConsumerResult] | None = None, release_archive_ownership: Callable[[], None], ) -> DurableChangeTrainExecution: - """Execute the real maintenance route through every persisted train state.""" + """Execute every persisted train state while the caller holds archive ownership. + + The caller-held lease must cover startup reconciliation and receipt creation so + reused forward-version evidence cannot become stale between those operations. + """ forward_version_evidence: dict[ArchiveTier, _DurableForwardVersionEvidence] = {} reconcile_durable_change_train_startup(archive_root, live_evidence_cache=forward_version_evidence) tier_path = archive_root / f"{tier.value}.db" @@ -1899,7 +1908,10 @@ def reconcile_durable_change_train_startup( *, live_evidence_cache: dict[ArchiveTier, _DurableForwardVersionEvidence] | None = None, ) -> tuple[Path, ...]: - """Reconcile backup-authorized trains left by a crashed maintenance process.""" + """Reconcile interrupted trains while the caller holds archive ownership. + + The caller-held lease must cover any subsequent use of ``live_evidence_cache``. + """ from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation with OwnedArchiveLocation.acquire( @@ -1928,6 +1940,7 @@ def _reconcile_durable_change_train_startup_locked( live_integrity_by_tier: dict[ArchiveTier, tuple[str, ...]] = {} live_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} canonical_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} + manifests_by_tier: dict[ArchiveTier, dict[int, DurableChangeTrain]] = {} manifest_paths = tuple(sorted(manifest_root.glob("*.json"))) def record_reconciled(path: Path) -> None: @@ -1968,6 +1981,7 @@ def record_reconciled(path: Path) -> None: _persist_train_transition(manifest_path, exc.failed_train, expected_revision=train.revision) raise train = _persist_train_transition(manifest_path, recovered, expected_revision=train.revision) + record_reconciled(manifest_path) if train.state in { DurableChangeTrainState.APPLIED, @@ -1986,9 +2000,11 @@ def record_reconciled(path: Path) -> None: actual = capture_durable_database_evidence(live, train.tier) live_evidence_by_tier[train.tier] = actual if actual.user_version > train.target_version: + if train.tier not in manifests_by_tier: + manifests_by_tier[train.tier] = _released_train_manifests_by_target(manifest_root, train.tier) _require_released_train_chain( train.tier, - _released_train_manifests_by_target(manifest_root, train.tier), + manifests_by_tier[train.tier], current_version=actual.user_version, ) if train.tier not in live_integrity_by_tier: diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index a550d7773c..ac3bb4dd88 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -1596,6 +1596,7 @@ def test_migrate_tier_cli_exposes_forward_version_receipt( ) assert plain.exit_code == 0, plain.output assert "historical train train:source:v2 is admitted at live schema v3" in plain.output + assert "(target v3)" in plain.output def test_migrate_tier_cli_refuses_live_daemon_before_sql( diff --git a/tests/unit/daemon/test_raw_failure_sample.py b/tests/unit/daemon/test_raw_failure_sample.py index 90ad753e77..03e51c7a51 100644 --- a/tests/unit/daemon/test_raw_failure_sample.py +++ b/tests/unit/daemon/test_raw_failure_sample.py @@ -605,11 +605,53 @@ def open_traced_readonly(path: Path) -> sqlite3.Connection: assert snapshot.parse_failures == snapshot.unexplained == 200 assert len(snapshot.samples) == 3 summary_queries = [statement for statement in statements if "GROUP BY f.origin" in statement] - sample_queries = [statement for statement in statements if "FROM sampled AS f" in statement] + sample_queries = [statement for statement in statements if "FROM sampled" in statement] assert len(summary_queries) == 1 assert len(sample_queries) == 1 assert "ROW_NUMBER()" not in sample_queries[0] assert "NOT EXISTS" in sample_queries[0] + assert "LIMIT 3" in sample_queries[0] + + def test_lifecycle_samples_are_sql_bounded_without_artifact_table( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + source_db = tmp_path / "source.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + with sqlite3.connect(source_db) as conn: + conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, + blob_size, acquired_at_ms, parse_error + ) VALUES (?, 'codex-session', ?, '/data/no-artifacts.jsonl', ?, ?, 0, ?, 'bad input') + """, + [ + (f"raw-{index}", f"native-{index}", index, bytes(32), 1_770_000_000_000 + index) + for index in range(5) + ], + ) + conn.execute("DROP TABLE raw_artifacts") + conn.commit() + + statements: list[str] = [] + + def open_traced_readonly(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(f"file:{path.resolve()}?mode=ro", uri=True) + connection.set_trace_callback(statements.append) + return connection + + monkeypatch.setattr("polylogue.storage.raw_failure_lifecycle.open_readonly_connection", open_traced_readonly) + snapshot = read_raw_failure_lifecycle(source_db, sample_limit=3) + + assert snapshot.parse_failures == snapshot.unexplained == 5 + assert snapshot.deferred == snapshot.terminal == 0 + assert len(snapshot.samples) == 3 + assert all(sample["artifact_kind"] is None and sample["support_status"] is None for sample in snapshot.samples) + sample_queries = [statement for statement in statements if "ORDER BY acquired_at_ms DESC" in statement] + assert len(sample_queries) == 1 + assert "LIMIT 3" in sample_queries[0] @pytest.mark.parametrize("source_state", ["missing", "malformed"]) def test_status_fails_closed_when_source_lifecycle_is_unavailable( diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 87fc82dd8c..043a265b85 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -475,6 +475,10 @@ def record_refresh_fsync(path: Path) -> None: assert released.apply_evidence is not None assert refreshed.apply_evidence.pre == released.apply_evidence.pre assert refreshed.apply_evidence.post.archive_identity_digest != released.apply_evidence.post.archive_identity_digest + assert ( + refreshed.apply_evidence.post.archive_identity_digest + == refreshed.source_continuity_evidence.archive_identity_digest + ) assert refreshed.proof == released.proof assert refreshed.revision == released.revision + 1 assert any(ref.startswith("proof:source-continuity-refresh:") for ref in refreshed.proof_refs) @@ -1190,8 +1194,9 @@ def count_canonical_inventories(tier: ArchiveTier, target_version: int) -> migra assert third.forward_version_receipt.historical_target_version == 2 assert third.forward_version_receipt.observed_live_version == 3 assert evidence_captures == 1 - # One live inventory plus the canonical inventory built from the fresh DDL. - # The no-op receipt reuses both instead of constructing a second pair. + # Three inventories: one nested inside the single evidence capture, one + # live inventory read during startup reconciliation, and one canonical + # inventory built from the fresh DDL. The no-op receipt reuses all of them. assert schema_inventories == 3 assert canonical_inventories == 1 @@ -1311,6 +1316,38 @@ def test_released_train_chain_is_anchored_at_adoption_floor() -> None: ) +def test_forward_receipt_checks_missing_chain_before_empty_history( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + floor = DURABLE_MIGRATION_ADOPTION_FLOORS[ArchiveTier.SOURCE] + current_version = floor + 2 + current_train = cast( + DurableChangeTrain, + SimpleNamespace(target_version=current_version, state=DurableChangeTrainState.RELEASED), + ) + monkeypatch.setattr( + durable_change_train_module, + "_released_train_manifests_by_target", + lambda _manifest_root, _tier: {current_version: current_train}, + ) + + with ( + sqlite3.connect(":memory:") as conn, + pytest.raises( + DurableChangeTrainError, + match=rf"versions \[{floor + 1}\]", + ), + ): + durable_change_train_module._forward_version_receipt_for_current_tier( + tmp_path, + conn, + ArchiveTier.SOURCE, + current_version=current_version, + current_target_version=current_version, + ) + + def test_startup_recovers_later_train_before_released_chain_validation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From d9ff240617a6f07143fb125de4ad3b67187f5c95 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 23:20:48 +0200 Subject: [PATCH 24/32] fix(storage): validate current durable train chain Problem: startup continuity validation skipped the adoption-floor chain when the only remaining released manifest matched the live version.\n\nWhat changed: run the cached released-train chain check whenever the live version exceeds the adoption floor, including equal-version current-train validation, and add a regression for a missing earlier train.\n\nVerification: devtools test tests/unit/storage/test_durable_change_train.py (48 passed).\n\nCo-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 3 +- .../unit/storage/test_durable_change_train.py | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index e28b360b4b..2e9b458280 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -1999,7 +1999,7 @@ def record_reconciled(path: Path) -> None: if actual is None: actual = capture_durable_database_evidence(live, train.tier) live_evidence_by_tier[train.tier] = actual - if actual.user_version > train.target_version: + if actual.user_version > DURABLE_MIGRATION_ADOPTION_FLOORS[train.tier]: if train.tier not in manifests_by_tier: manifests_by_tier[train.tier] = _released_train_manifests_by_target(manifest_root, train.tier) _require_released_train_chain( @@ -2007,6 +2007,7 @@ def record_reconciled(path: Path) -> None: manifests_by_tier[train.tier], current_version=actual.user_version, ) + if actual.user_version > train.target_version: if train.tier not in live_integrity_by_tier: live_integrity_by_tier[train.tier] = tuple( str(row[0]) for row in live.execute("PRAGMA integrity_check") diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 043a265b85..bc82874a07 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1426,6 +1426,42 @@ def fake_capture(*_args: object, **_kwargs: object) -> SimpleNamespace: assert events == [("recover", 28), ("verify", 27), ("verify", 28)] +def test_startup_checks_chain_when_only_current_train_remains( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + manifest_root = tmp_path / ".maintenance-state" / "durable-change-trains" + manifest_root.mkdir(parents=True) + manifest_path = manifest_root / "source-028.json" + manifest_path.touch() + current = cast( + DurableChangeTrain, + SimpleNamespace(state=DurableChangeTrainState.RELEASED, tier=ArchiveTier.SOURCE, target_version=28), + ) + + @contextmanager + def fake_open_tier(_path: Path) -> Iterator[sqlite3.Connection]: + with sqlite3.connect(":memory:") as connection: + yield connection + + monkeypatch.setattr(durable_change_train_module, "_recover_pending_source_continuity_intents", lambda _root: None) + monkeypatch.setattr(durable_change_train_module, "_open_existing_tier", fake_open_tier) + monkeypatch.setattr(durable_change_train_module, "load_durable_change_train_manifest", lambda _path: current) + monkeypatch.setattr( + durable_change_train_module, + "capture_durable_database_evidence", + lambda _connection, _tier: SimpleNamespace(user_version=28), + ) + monkeypatch.setattr( + durable_change_train_module, + "_released_train_manifests_by_target", + lambda _root, _tier: {28: current}, + ) + + with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): + durable_change_train_module._reconcile_durable_change_train_startup_locked(tmp_path) + + def test_source_train_identity_survives_late_user_tier_initialization(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database From c0ab48a33905070e09f685ea484769f219b41ec2 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 23:33:45 +0200 Subject: [PATCH 25/32] fix(storage): reject manifestless durable tiers Problem: startup admitted an existing durable tier above the adoption floor when the durable change-train manifest directory was absent or empty.\n\nWhat changed: enforce the released-train chain after lifecycle recovery so manifestless tiers fail closed without masking persisted indeterminate failures, and add a regression test.\n\nVerification: direnv exec . devtools test tests/unit/storage/test_durable_change_train.py (49 passed).\n\nCo-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 22 ++++++++++++++-- .../unit/storage/test_durable_change_train.py | 25 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 2e9b458280..0169133d90 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -1933,8 +1933,6 @@ def _reconcile_durable_change_train_startup_locked( """Reconcile persisted trains while the caller holds archive ownership.""" _recover_pending_source_continuity_intents(archive_root) manifest_root = archive_root / ".maintenance-state" / "durable-change-trains" - if not manifest_root.is_dir(): - return () reconciled: list[Path] = [] live_evidence_by_tier: dict[ArchiveTier, DurableDatabaseEvidence] = {} live_integrity_by_tier: dict[ArchiveTier, tuple[str, ...]] = {} @@ -1990,6 +1988,26 @@ def record_reconciled(path: Path) -> None: _prove_and_release_persisted_train(archive_root, manifest_path, train) record_reconciled(manifest_path) + # An existing durable tier above its adoption floor must have a complete + # released-train chain even when the manifest directory is absent or empty. + # Fresh database creation does not enter this startup reconciliation route. + # Recovery runs first so an indeterminate persisted failure keeps its + # stronger fail-closed error rather than being masked by chain validation. + for tier, adoption_floor in DURABLE_MIGRATION_ADOPTION_FLOORS.items(): + tier_path = archive_root / f"{tier.value}.db" + if not tier_path.is_file(): + continue + with _open_existing_tier(tier_path) as live: + current_version = int(live.execute("PRAGMA user_version").fetchone()[0] or 0) + if current_version <= adoption_floor: + continue + manifests_by_tier[tier] = _released_train_manifests_by_target(manifest_root, tier) + _require_released_train_chain( + tier, + manifests_by_tier[tier], + current_version=current_version, + ) + for manifest_path in manifest_paths: train = load_durable_change_train_manifest(manifest_path) if train.state is not DurableChangeTrainState.RELEASED: diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index bc82874a07..14e9395bf5 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1398,7 +1398,11 @@ def fake_recover(*_args: object, **_kwargs: object) -> DurableChangeTrain: def fake_capture(*_args: object, **_kwargs: object) -> SimpleNamespace: return SimpleNamespace(user_version=28) - monkeypatch.setattr(durable_change_train_module, "_recover_pending_source_continuity_intents", lambda _root: None) + monkeypatch.setattr( + durable_change_train_module, + "_recover_pending_source_continuity_intents", + lambda _root: None, + ) monkeypatch.setattr(durable_change_train_module, "_open_existing_tier", fake_open_tier) monkeypatch.setattr(durable_change_train_module, "load_durable_change_train_manifest", fake_load) monkeypatch.setattr(durable_change_train_module, "_persist_train_transition", fake_persist) @@ -1462,6 +1466,25 @@ def fake_open_tier(_path: Path) -> Iterator[sqlite3.Connection]: durable_change_train_module._reconcile_durable_change_train_startup_locked(tmp_path) +def test_startup_checks_chain_when_manifest_directory_is_missing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "source.db").touch() + + @contextmanager + def fake_open_tier(_path: Path) -> Iterator[sqlite3.Connection]: + with sqlite3.connect(":memory:") as connection: + connection.execute("PRAGMA user_version = 28") + yield connection + + monkeypatch.setattr(durable_change_train_module, "_recover_pending_source_continuity_intents", lambda _root: None) + monkeypatch.setattr(durable_change_train_module, "_open_existing_tier", fake_open_tier) + + with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): + durable_change_train_module._reconcile_durable_change_train_startup_locked(tmp_path) + + def test_source_train_identity_survives_late_user_tier_initialization(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database From 733e98efec753e97173d1ed58846b7c964b90524 Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 23:40:51 +0200 Subject: [PATCH 26/32] fix(storage): distinguish fresh durable bootstrap Problem: enforcing released train chains on every existing durable tier also rejected freshly bootstrapped archives that start at the current schema without migration history.\n\nWhat changed: record a version- and archive-identity-bound bootstrap receipt, accept that receipt only while the durable tier remains at its direct-bootstrap version with no train manifests, and keep manifestless legacy tiers fail-closed.\n\nVerification: direnv exec . devtools test tests/unit/storage/test_durable_change_train.py (50 passed).\n\nCo-Authored-By: Claude --- .../storage/sqlite/archive_tiers/bootstrap.py | 7 +++ .../storage/sqlite/durable_change_train.py | 55 +++++++++++++++++++ .../unit/storage/test_durable_change_train.py | 8 +++ 3 files changed, 70 insertions(+) diff --git a/polylogue/storage/sqlite/archive_tiers/bootstrap.py b/polylogue/storage/sqlite/archive_tiers/bootstrap.py index 6353bf310f..503ac31608 100644 --- a/polylogue/storage/sqlite/archive_tiers/bootstrap.py +++ b/polylogue/storage/sqlite/archive_tiers/bootstrap.py @@ -313,6 +313,9 @@ def initialize_active_archive_root(root: Path) -> None: """Create or initialize every tier database in an archive root.""" from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation + fresh_durable_bootstrap = not any( + (root / archive_tier_spec(tier).filename).exists() for tier in DURABLE_MIGRATION_TIERS + ) with OwnedArchiveLocation.acquire( ArchiveLocation.resolve(root), owner_id=f"bootstrap:{os.getpid()}", @@ -321,6 +324,10 @@ def initialize_active_archive_root(root: Path) -> None: reconcile_durable_change_trains_on_startup(root) for spec in ARCHIVE_TIER_SPECS.values(): initialize_archive_database(root / spec.filename, spec.tier) + if fresh_durable_bootstrap: + from polylogue.storage.sqlite.durable_change_train import _record_fresh_durable_bootstrap + + _record_fresh_durable_bootstrap(root) def reconcile_durable_change_trains_on_startup(root: Path) -> tuple[Path, ...]: diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 0169133d90..437edb847c 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -74,6 +74,8 @@ _MIGRATION_NAME_RE = re.compile(r"^(?P\d{3,})_[a-z0-9_]+\.sql$") _DROP_SQL_RE = re.compile(r"(?is)\bDROP\s+(?:TABLE|INDEX|TRIGGER|VIEW)\b") _SOURCE_CONTINUITY_PENDING_FORMAT = "polylogue.source-continuity-pending.v1" +_FRESH_DURABLE_BOOTSTRAP_FORMAT = "polylogue.durable-bootstrap.v1" +_FRESH_DURABLE_BOOTSTRAP_MARKER = ".bootstrap" class DurableSourceTrainMissingError(DurableChangeTrainError): @@ -318,6 +320,55 @@ def durable_change_train_manifest_path(archive_root: Path, tier: ArchiveTier, sl return archive_root / ".maintenance-state" / "durable-change-trains" / f"{tier.value}-{slot:03d}.json" +def _record_fresh_durable_bootstrap(archive_root: Path) -> None: + """Record the versions and identity of a direct, current-schema bootstrap.""" + from polylogue.storage.archive_identity import ArchiveIdentity + + archive_root = archive_root.resolve() + marker_root = archive_root / ".maintenance-state" / "durable-change-trains" + marker_root.mkdir(parents=True, exist_ok=True) + versions: dict[str, int] = {} + for tier in DURABLE_MIGRATION_ADOPTION_FLOORS: + with sqlite3.connect(archive_root / f"{tier.value}.db") as connection: + versions[tier.value] = int(connection.execute("PRAGMA user_version").fetchone()[0]) + payload = { + "format": _FRESH_DURABLE_BOOTSTRAP_FORMAT, + "archive_identity_digest": ArchiveIdentity.resolve(archive_root).authority_identity_digest, + "versions": versions, + } + marker_root.joinpath(_FRESH_DURABLE_BOOTSTRAP_MARKER).write_text( + json.dumps(payload, sort_keys=True, separators=(",", ":")), + encoding="utf-8", + ) + + +def _fresh_durable_bootstrap_versions(archive_root: Path, marker_root: Path) -> dict[ArchiveTier, int]: + """Return direct-bootstrap versions when the marker is authentic.""" + from polylogue.storage.archive_identity import ArchiveIdentity + + marker_path = marker_root / _FRESH_DURABLE_BOOTSTRAP_MARKER + if not marker_path.is_file(): + return {} + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise DurableChangeTrainError(f"invalid fresh durable bootstrap marker: {marker_path}") from exc + if not isinstance(payload, dict) or payload.get("format") != _FRESH_DURABLE_BOOTSTRAP_FORMAT: + raise DurableChangeTrainError(f"fresh durable bootstrap marker format mismatch: {marker_path}") + if payload.get("archive_identity_digest") != ArchiveIdentity.resolve(archive_root).authority_identity_digest: + raise DurableChangeTrainError("fresh durable bootstrap marker archive identity mismatch") + raw_versions = payload.get("versions") + if not isinstance(raw_versions, dict): + raise DurableChangeTrainError(f"fresh durable bootstrap marker versions are invalid: {marker_path}") + versions: dict[ArchiveTier, int] = {} + for tier in DURABLE_MIGRATION_ADOPTION_FLOORS: + raw_version = raw_versions.get(tier.value) + if not isinstance(raw_version, int) or raw_version < 0: + raise DurableChangeTrainError(f"fresh durable bootstrap marker version is invalid: {marker_path}") + versions[tier] = raw_version + return versions + + def durable_migration_sidecar_for_slot(tier: ArchiveTier, slot: int) -> DurableMigrationSidecar | None: """Load the package sidecar for the next numbered production migration.""" if tier not in DURABLE_MIGRATION_ADOPTION_FLOORS: @@ -1940,6 +1991,7 @@ def _reconcile_durable_change_train_startup_locked( canonical_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} manifests_by_tier: dict[ArchiveTier, dict[int, DurableChangeTrain]] = {} manifest_paths = tuple(sorted(manifest_root.glob("*.json"))) + fresh_bootstrap_versions = _fresh_durable_bootstrap_versions(archive_root, manifest_root) def record_reconciled(path: Path) -> None: if path not in reconciled: @@ -2002,6 +2054,9 @@ def record_reconciled(path: Path) -> None: if current_version <= adoption_floor: continue manifests_by_tier[tier] = _released_train_manifests_by_target(manifest_root, tier) + tier_manifest_paths = tuple(manifest_root.glob(f"{tier.value}-*.json")) + if fresh_bootstrap_versions.get(tier) == current_version and not tier_manifest_paths: + continue _require_released_train_chain( tier, manifests_by_tier[tier], diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 14e9395bf5..c79f2b0c15 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1485,6 +1485,14 @@ def fake_open_tier(_path: Path) -> Iterator[sqlite3.Connection]: durable_change_train_module._reconcile_durable_change_train_startup_locked(tmp_path) +def test_fresh_archive_bootstrap_receipt_allows_repeat_startup(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + assert reconcile_durable_change_train_startup(tmp_path) == () + initialize_active_archive_root(tmp_path) + + def test_source_train_identity_survives_late_user_tier_initialization(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database From bc8fb78929662c74e3b02feb9676fb3a729a59ac Mon Sep 17 00:00:00 2001 From: Sinity Date: Fri, 7 Aug 2026 23:46:09 +0200 Subject: [PATCH 27/32] test(storage): cover bootstrap receipt identity Problem: the fresh-bootstrap exception must remain bound to the archive it authenticated.\n\nWhat changed: add regression coverage proving a tampered bootstrap receipt is rejected during startup reconciliation.\n\nVerification: direnv exec . devtools test tests/unit/storage/test_durable_change_train.py (51 passed).\n\nCo-Authored-By: Claude --- tests/unit/storage/test_durable_change_train.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index c79f2b0c15..720bf28801 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1493,6 +1493,19 @@ def test_fresh_archive_bootstrap_receipt_allows_repeat_startup(tmp_path: Path) - initialize_active_archive_root(tmp_path) +def test_fresh_bootstrap_receipt_rejects_archive_identity_mismatch(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + marker = tmp_path / ".maintenance-state" / "durable-change-trains" / ".bootstrap" + payload = json.loads(marker.read_text(encoding="utf-8")) + payload["archive_identity_digest"] = "0" * 64 + marker.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(DurableChangeTrainError, match="archive identity mismatch"): + reconcile_durable_change_train_startup(tmp_path) + + def test_source_train_identity_survives_late_user_tier_initialization(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database From ec37aecfe9c6fb630e77333afb5557493c2f44c5 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 00:01:15 +0200 Subject: [PATCH 28/32] fix(storage): authenticate bootstrap adoption Problem: the bootstrap receipt exception rejected pre-marker current-schema archives, depended on rebuildable index identity, and restarted train chains at the global adoption floor after the first migration.\n\nWhat changed: authenticate a one-time pre-marker adoption against current canonical durable schemas, bind receipts to source/user identity, and use the recorded bootstrap version as the chain floor for later migrations. Missing or replaced durable tiers remain fail-closed.\n\nVerification: direnv exec . devtools test tests/unit/storage/test_durable_change_train.py (54 passed).\n\nCo-Authored-By: Claude --- .../storage/sqlite/archive_tiers/bootstrap.py | 20 ++++- .../storage/sqlite/durable_change_train.py | 79 +++++++++++++++++-- .../unit/storage/test_durable_change_train.py | 36 ++++++++- 3 files changed, 123 insertions(+), 12 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/bootstrap.py b/polylogue/storage/sqlite/archive_tiers/bootstrap.py index 503ac31608..417b408443 100644 --- a/polylogue/storage/sqlite/archive_tiers/bootstrap.py +++ b/polylogue/storage/sqlite/archive_tiers/bootstrap.py @@ -313,21 +313,35 @@ def initialize_active_archive_root(root: Path) -> None: """Create or initialize every tier database in an archive root.""" from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation - fresh_durable_bootstrap = not any( - (root / archive_tier_spec(tier).filename).exists() for tier in DURABLE_MIGRATION_TIERS + durable_tier_exists = any((root / archive_tier_spec(tier).filename).exists() for tier in DURABLE_MIGRATION_TIERS) + manifest_root = root / ".maintenance-state" / "durable-change-trains" + has_durable_train_state = any(manifest_root.glob("*.json")) + has_bootstrap_marker = (manifest_root / ".bootstrap").is_file() + fresh_durable_bootstrap = not durable_tier_exists and not has_durable_train_state and not has_bootstrap_marker + pre_marker_adoption = ( + (root / archive_tier_spec(ArchiveTier.SOURCE).filename).is_file() + and durable_tier_exists + and not has_durable_train_state + and not has_bootstrap_marker ) with OwnedArchiveLocation.acquire( ArchiveLocation.resolve(root), owner_id=f"bootstrap:{os.getpid()}", allow_reentrant=True, ): - reconcile_durable_change_trains_on_startup(root) + if not fresh_durable_bootstrap and not pre_marker_adoption: + reconcile_durable_change_trains_on_startup(root) for spec in ARCHIVE_TIER_SPECS.values(): initialize_archive_database(root / spec.filename, spec.tier) if fresh_durable_bootstrap: from polylogue.storage.sqlite.durable_change_train import _record_fresh_durable_bootstrap _record_fresh_durable_bootstrap(root) + elif pre_marker_adoption: + from polylogue.storage.sqlite.durable_change_train import _adopt_pre_marker_durable_bootstrap + + _adopt_pre_marker_durable_bootstrap(root) + reconcile_durable_change_trains_on_startup(root) def reconcile_durable_change_trains_on_startup(root: Path) -> tuple[Path, ...]: diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 437edb847c..5fda7166c6 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -333,7 +333,7 @@ def _record_fresh_durable_bootstrap(archive_root: Path) -> None: versions[tier.value] = int(connection.execute("PRAGMA user_version").fetchone()[0]) payload = { "format": _FRESH_DURABLE_BOOTSTRAP_FORMAT, - "archive_identity_digest": ArchiveIdentity.resolve(archive_root).authority_identity_digest, + "durable_identity_digest": _durable_identity_digest(ArchiveIdentity.resolve(archive_root)), "versions": versions, } marker_root.joinpath(_FRESH_DURABLE_BOOTSTRAP_MARKER).write_text( @@ -355,8 +355,8 @@ def _fresh_durable_bootstrap_versions(archive_root: Path, marker_root: Path) -> raise DurableChangeTrainError(f"invalid fresh durable bootstrap marker: {marker_path}") from exc if not isinstance(payload, dict) or payload.get("format") != _FRESH_DURABLE_BOOTSTRAP_FORMAT: raise DurableChangeTrainError(f"fresh durable bootstrap marker format mismatch: {marker_path}") - if payload.get("archive_identity_digest") != ArchiveIdentity.resolve(archive_root).authority_identity_digest: - raise DurableChangeTrainError("fresh durable bootstrap marker archive identity mismatch") + if payload.get("durable_identity_digest") != _durable_identity_digest(ArchiveIdentity.resolve(archive_root)): + raise DurableChangeTrainError("fresh durable bootstrap marker durable identity mismatch") raw_versions = payload.get("versions") if not isinstance(raw_versions, dict): raise DurableChangeTrainError(f"fresh durable bootstrap marker versions are invalid: {marker_path}") @@ -369,6 +369,49 @@ def _fresh_durable_bootstrap_versions(archive_root: Path, marker_root: Path) -> return versions +def _durable_identity_digest(identity: object) -> str: + """Digest only the durable source/user identity for bootstrap receipts.""" + from polylogue.storage.archive_identity import ArchiveIdentity + + if not isinstance(identity, ArchiveIdentity): + raise TypeError("durable identity digest requires an ArchiveIdentity") + payload = { + "configured_root": str(identity.configured_root.absolute()), + "durable_id": identity.durable_id, + } + return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def _adopt_pre_marker_durable_bootstrap(archive_root: Path) -> None: + """Authenticate a current-schema archive created before bootstrap receipts.""" + from polylogue.storage.sqlite.archive_tiers import ARCHIVE_VERSION_BY_TIER + + archive_root = archive_root.resolve() + manifest_root = archive_root / ".maintenance-state" / "durable-change-trains" + if (manifest_root / _FRESH_DURABLE_BOOTSTRAP_MARKER).is_file(): + return + if any(manifest_root.glob("*.json")): + return + for tier in DURABLE_MIGRATION_ADOPTION_FLOORS: + tier_path = archive_root / f"{tier.value}.db" + if not tier_path.is_file(): + continue + with _open_existing_tier(tier_path) as connection: + current_version = int(connection.execute("PRAGMA user_version").fetchone()[0] or 0) + expected_version = ARCHIVE_VERSION_BY_TIER[tier] + if current_version != expected_version: + raise DurableChangeTrainError( + f"pre-marker {tier.value} durable tier is v{current_version}, expected current v{expected_version}" + ) + actual_inventory = _migration_runner.capture_durable_schema_inventory(connection) + expected_inventory = _canonical_schema_inventory(tier, expected_version) + if actual_inventory.sha256 != expected_inventory.sha256: + raise DurableChangeTrainError( + f"pre-marker {tier.value} durable tier schema does not match current canonical DDL" + ) + _record_fresh_durable_bootstrap(archive_root) + + def durable_migration_sidecar_for_slot(tier: ArchiveTier, slot: int) -> DurableMigrationSidecar | None: """Load the package sidecar for the next numbered production migration.""" if tier not in DURABLE_MIGRATION_ADOPTION_FLOORS: @@ -1648,6 +1691,12 @@ def _forward_version_receipt_for_current_tier( tier, manifests_by_target, current_version=current_version, + floor=max( + DURABLE_MIGRATION_ADOPTION_FLOORS[tier], + _fresh_durable_bootstrap_versions(archive_root, manifest_root).get( + tier, DURABLE_MIGRATION_ADOPTION_FLOORS[tier] + ), + ), ) if not historical: return None @@ -1698,21 +1747,27 @@ def _require_released_train_chain( manifests_by_target: dict[int, DurableChangeTrain], *, current_version: int, + floor: int | None = None, ) -> None: """Require released, schema-proven evidence for every later version.""" + chain_floor = DURABLE_MIGRATION_ADOPTION_FLOORS[tier] if floor is None else floor + if chain_floor < DURABLE_MIGRATION_ADOPTION_FLOORS[tier] or chain_floor > current_version: + raise DurableChangeTrainError( + f"invalid {tier.value} durable train chain floor v{chain_floor} for live v{current_version}" + ) missing_targets = [ version - for version in range(DURABLE_MIGRATION_ADOPTION_FLOORS[tier] + 1, current_version + 1) + for version in range(chain_floor + 1, current_version + 1) if manifests_by_target.get(version) is None or manifests_by_target[version].state is not DurableChangeTrainState.RELEASED ] if missing_targets: raise DurableChangeTrainError( f"{tier.value} durable forward admission lacks released train evidence for versions " - f"{missing_targets} from adoption floor v{DURABLE_MIGRATION_ADOPTION_FLOORS[tier]} " + f"{missing_targets} from chain floor v{chain_floor} " f"through live v{current_version}" ) - for version in range(DURABLE_MIGRATION_ADOPTION_FLOORS[tier] + 1, current_version + 1): + for version in range(chain_floor + 1, current_version + 1): _historical_schema_evidence(manifests_by_target[version]) @@ -2055,12 +2110,18 @@ def record_reconciled(path: Path) -> None: continue manifests_by_tier[tier] = _released_train_manifests_by_target(manifest_root, tier) tier_manifest_paths = tuple(manifest_root.glob(f"{tier.value}-*.json")) - if fresh_bootstrap_versions.get(tier) == current_version and not tier_manifest_paths: + bootstrap_version = fresh_bootstrap_versions.get(tier) + if bootstrap_version is not None and current_version < bootstrap_version: + raise DurableChangeTrainError( + f"{tier.value} durable tier regressed below fresh bootstrap v{bootstrap_version}" + ) + if bootstrap_version == current_version and not tier_manifest_paths: continue _require_released_train_chain( tier, manifests_by_tier[tier], current_version=current_version, + floor=max(adoption_floor, bootstrap_version or adoption_floor), ) for manifest_path in manifest_paths: @@ -2079,6 +2140,10 @@ def record_reconciled(path: Path) -> None: train.tier, manifests_by_tier[train.tier], current_version=actual.user_version, + floor=max( + DURABLE_MIGRATION_ADOPTION_FLOORS[train.tier], + fresh_bootstrap_versions.get(train.tier, DURABLE_MIGRATION_ADOPTION_FLOORS[train.tier]), + ), ) if actual.user_version > train.target_version: if train.tier not in live_integrity_by_tier: diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 720bf28801..4b3eea12c8 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1316,6 +1316,18 @@ def test_released_train_chain_is_anchored_at_adoption_floor() -> None: ) +def test_released_train_chain_can_start_at_bootstrap_floor(monkeypatch: pytest.MonkeyPatch) -> None: + released = cast(DurableChangeTrain, SimpleNamespace(state=DurableChangeTrainState.RELEASED)) + monkeypatch.setattr(durable_change_train_module, "_historical_schema_evidence", lambda _train: None) + + durable_change_train_module._require_released_train_chain( + ArchiveTier.SOURCE, + {30: released}, + current_version=30, + floor=29, + ) + + def test_forward_receipt_checks_missing_chain_before_empty_history( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1493,16 +1505,36 @@ def test_fresh_archive_bootstrap_receipt_allows_repeat_startup(tmp_path: Path) - initialize_active_archive_root(tmp_path) +def test_pre_marker_current_archive_is_adopted_once(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + marker = tmp_path / ".maintenance-state" / "durable-change-trains" / ".bootstrap" + marker.unlink() + + initialize_active_archive_root(tmp_path) + assert marker.is_file() + + +def test_bootstrap_marker_survives_index_generation_replacement(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + (tmp_path / "index.db").unlink() + + initialize_active_archive_root(tmp_path) + + def test_fresh_bootstrap_receipt_rejects_archive_identity_mismatch(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root initialize_active_archive_root(tmp_path) marker = tmp_path / ".maintenance-state" / "durable-change-trains" / ".bootstrap" payload = json.loads(marker.read_text(encoding="utf-8")) - payload["archive_identity_digest"] = "0" * 64 + payload["durable_identity_digest"] = "0" * 64 marker.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(DurableChangeTrainError, match="archive identity mismatch"): + with pytest.raises(DurableChangeTrainError, match="durable identity mismatch"): reconcile_durable_change_train_startup(tmp_path) From 06788ed31eead494ff2ada761a86e133bbd16302 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 00:24:09 +0200 Subject: [PATCH 29/32] fix(storage): close durable train review gaps Problem The durable train review found races and unauthenticated bootstrap versions. Raw failure status also selected a different sample set from the lifecycle projection. What changed Move bootstrap classification under the archive ownership lock, authenticate marker versions, centralize chain floors, reuse cached forward evidence, and preserve per-manifest validation. Make status consume the lifecycle-selected raw IDs and order classified failures before unexplained rows. Verification Ref #3875 Passed: direnv exec . devtools test tests/unit/daemon/test_raw_failure_sample.py tests/unit/storage/test_durable_change_train.py tests/unit/cli/test_archive_maintenance_cli.py::test_migrate_tier_cli_exposes_forward_version_receipt Result: 94 passed. Co-Authored-By: Claude --- polylogue/daemon/status.py | 31 ++++++--- polylogue/storage/raw_failure_lifecycle.py | 2 +- .../storage/sqlite/archive_tiers/bootstrap.py | 26 ++++--- .../storage/sqlite/durable_change_train.py | 59 ++++++++++++---- tests/unit/daemon/test_raw_failure_sample.py | 67 ++++++++++++++++++- .../unit/storage/test_durable_change_train.py | 21 +++++- 6 files changed, 166 insertions(+), 40 deletions(-) diff --git a/polylogue/daemon/status.py b/polylogue/daemon/status.py index e1fd4c32bc..00c9204808 100644 --- a/polylogue/daemon/status.py +++ b/polylogue/daemon/status.py @@ -925,21 +925,32 @@ def _archive_raw_failure_info( ).fetchone()[0] or 0 ) + sample_ids = [ + str(sample["raw_id"]) for sample in lifecycle_snapshot.samples if sample.get("raw_id") is not None + ] lifecycle_by_raw_id = { str(sample["raw_id"]): str(sample["lifecycle"]) for sample in lifecycle_snapshot.samples if sample.get("raw_id") is not None and sample.get("lifecycle") is not None } samples: list[RawFailureSample] = [] - for row in conn.execute( - """ - SELECT r.raw_id, r.origin, r.parse_error, r.validation_status, r.validation_error - FROM raw_sessions AS r - WHERE (parse_error IS NOT NULL AND TRIM(parse_error) != '') OR validation_status = 'failed' - ORDER BY acquired_at_ms DESC, raw_id DESC - LIMIT 50 - """ - ): + rows_by_raw_id: dict[str, sqlite3.Row | tuple[object, ...]] = {} + if sample_ids: + placeholders = ",".join("?" for _ in sample_ids) + rows = conn.execute( + f""" + SELECT r.raw_id, r.origin, r.parse_error, r.validation_status, r.validation_error + FROM raw_sessions AS r + WHERE r.raw_id IN ({placeholders}) + AND ((r.parse_error IS NOT NULL AND TRIM(r.parse_error) != '') OR r.validation_status = 'failed') + """, + sample_ids, + ) + rows_by_raw_id = {str(row[0]): row for row in rows} + for raw_id in sample_ids: + row = rows_by_raw_id.get(raw_id) + if row is None: + continue parse_err = str(row[2] or "") if row[2] else "" val_status = str(row[3] or "") if row[3] else "" val_err = str(row[4] or "") if row[4] else "" @@ -959,7 +970,7 @@ def _archive_raw_failure_info( redacted_error=parse_err or val_err, lifecycle=cast( Literal["deferred", "terminal", "unexplained"], - lifecycle_by_raw_id.get(str(row[0]), "unexplained"), + lifecycle_by_raw_id.get(raw_id, "unexplained"), ), ) ) diff --git a/polylogue/storage/raw_failure_lifecycle.py b/polylogue/storage/raw_failure_lifecycle.py index 21a3ad1ff0..fa90cc8371 100644 --- a/polylogue/storage/raw_failure_lifecycle.py +++ b/polylogue/storage/raw_failure_lifecycle.py @@ -189,7 +189,7 @@ def read_raw_failure_lifecycle(source_db: Path, *, sample_limit: int = 10) -> Ra ('terminal_corrupt_input', 'decode_failed'), ('terminal_unsupported_shape', 'unsupported_parseable') ) THEN 1 - ELSE 0 + ELSE 2 END, f.acquired_at_ms DESC, f.raw_id DESC LIMIT ? diff --git a/polylogue/storage/sqlite/archive_tiers/bootstrap.py b/polylogue/storage/sqlite/archive_tiers/bootstrap.py index 417b408443..49d5c71cee 100644 --- a/polylogue/storage/sqlite/archive_tiers/bootstrap.py +++ b/polylogue/storage/sqlite/archive_tiers/bootstrap.py @@ -313,22 +313,26 @@ def initialize_active_archive_root(root: Path) -> None: """Create or initialize every tier database in an archive root.""" from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation - durable_tier_exists = any((root / archive_tier_spec(tier).filename).exists() for tier in DURABLE_MIGRATION_TIERS) - manifest_root = root / ".maintenance-state" / "durable-change-trains" - has_durable_train_state = any(manifest_root.glob("*.json")) - has_bootstrap_marker = (manifest_root / ".bootstrap").is_file() - fresh_durable_bootstrap = not durable_tier_exists and not has_durable_train_state and not has_bootstrap_marker - pre_marker_adoption = ( - (root / archive_tier_spec(ArchiveTier.SOURCE).filename).is_file() - and durable_tier_exists - and not has_durable_train_state - and not has_bootstrap_marker - ) with OwnedArchiveLocation.acquire( ArchiveLocation.resolve(root), owner_id=f"bootstrap:{os.getpid()}", allow_reentrant=True, ): + # Classify the archive after acquiring ownership. Another process may + # publish a marker or durable train while the probe is in flight. + durable_tier_exists = any( + (root / archive_tier_spec(tier).filename).exists() for tier in DURABLE_MIGRATION_TIERS + ) + manifest_root = root / ".maintenance-state" / "durable-change-trains" + has_durable_train_state = any(manifest_root.glob("*.json")) + has_bootstrap_marker = (manifest_root / ".bootstrap").is_file() + fresh_durable_bootstrap = not durable_tier_exists and not has_durable_train_state and not has_bootstrap_marker + pre_marker_adoption = ( + (root / archive_tier_spec(ArchiveTier.SOURCE).filename).is_file() + and durable_tier_exists + and not has_durable_train_state + and not has_bootstrap_marker + ) if not fresh_durable_bootstrap and not pre_marker_adoption: reconcile_durable_change_trains_on_startup(root) for spec in ARCHIVE_TIER_SPECS.values(): diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 5fda7166c6..dccb956dea 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -326,16 +326,19 @@ def _record_fresh_durable_bootstrap(archive_root: Path) -> None: archive_root = archive_root.resolve() marker_root = archive_root / ".maintenance-state" / "durable-change-trains" + if (marker_root / _FRESH_DURABLE_BOOTSTRAP_MARKER).exists() or any(marker_root.glob("*.json")): + raise DurableChangeTrainError(f"cannot record fresh durable bootstrap over existing train state: {marker_root}") marker_root.mkdir(parents=True, exist_ok=True) versions: dict[str, int] = {} for tier in DURABLE_MIGRATION_ADOPTION_FLOORS: with sqlite3.connect(archive_root / f"{tier.value}.db") as connection: versions[tier.value] = int(connection.execute("PRAGMA user_version").fetchone()[0]) - payload = { + payload: dict[str, object] = { "format": _FRESH_DURABLE_BOOTSTRAP_FORMAT, "durable_identity_digest": _durable_identity_digest(ArchiveIdentity.resolve(archive_root)), "versions": versions, } + payload["marker_digest"] = _bootstrap_marker_digest(payload) marker_root.joinpath(_FRESH_DURABLE_BOOTSTRAP_MARKER).write_text( json.dumps(payload, sort_keys=True, separators=(",", ":")), encoding="utf-8", @@ -346,6 +349,8 @@ def _fresh_durable_bootstrap_versions(archive_root: Path, marker_root: Path) -> """Return direct-bootstrap versions when the marker is authentic.""" from polylogue.storage.archive_identity import ArchiveIdentity + archive_root = archive_root.resolve() + marker_root = archive_root / ".maintenance-state" / "durable-change-trains" marker_path = marker_root / _FRESH_DURABLE_BOOTSTRAP_MARKER if not marker_path.is_file(): return {} @@ -357,6 +362,11 @@ def _fresh_durable_bootstrap_versions(archive_root: Path, marker_root: Path) -> raise DurableChangeTrainError(f"fresh durable bootstrap marker format mismatch: {marker_path}") if payload.get("durable_identity_digest") != _durable_identity_digest(ArchiveIdentity.resolve(archive_root)): raise DurableChangeTrainError("fresh durable bootstrap marker durable identity mismatch") + marker_digest = payload.get("marker_digest") + unsigned_payload = dict(payload) + unsigned_payload.pop("marker_digest", None) + if not isinstance(marker_digest, str) or marker_digest != _bootstrap_marker_digest(unsigned_payload): + raise DurableChangeTrainError("fresh durable bootstrap marker digest mismatch") raw_versions = payload.get("versions") if not isinstance(raw_versions, dict): raise DurableChangeTrainError(f"fresh durable bootstrap marker versions are invalid: {marker_path}") @@ -382,6 +392,11 @@ def _durable_identity_digest(identity: object) -> str: return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest() +def _bootstrap_marker_digest(payload: dict[str, object]) -> str: + """Authenticate bootstrap identity and recorded durable versions together.""" + return _canonical_json_sha256(payload) + + def _adopt_pre_marker_durable_bootstrap(archive_root: Path) -> None: """Authenticate a current-schema archive created before bootstrap receipts.""" from polylogue.storage.sqlite.archive_tiers import ARCHIVE_VERSION_BY_TIER @@ -886,7 +901,9 @@ def _refresh_released_source_train_continuity_locked( caller must first validate the named operation and backup against the pre-mutation live tier. This helper then binds those exact receipt and backup bytes to a separate current-evidence record. The original migration - evidence remains immutable in ``apply_evidence``. + evidence remains immutable in ``apply_evidence``; only the + legacy-authority-digest compatibility path may rewrite that historical + identity field while preserving the migration evidence. """ from polylogue.storage.archive_identity import ArchiveIdentity, ArchiveLocation, OwnedArchiveLocation @@ -1691,12 +1708,7 @@ def _forward_version_receipt_for_current_tier( tier, manifests_by_target, current_version=current_version, - floor=max( - DURABLE_MIGRATION_ADOPTION_FLOORS[tier], - _fresh_durable_bootstrap_versions(archive_root, manifest_root).get( - tier, DURABLE_MIGRATION_ADOPTION_FLOORS[tier] - ), - ), + floor=_chain_floor(tier, _fresh_durable_bootstrap_versions(archive_root, manifest_root)), ) if not historical: return None @@ -1771,6 +1783,12 @@ def _require_released_train_chain( _historical_schema_evidence(manifests_by_target[version]) +def _chain_floor(tier: ArchiveTier, bootstrap_versions: dict[ArchiveTier, int]) -> int: + """Return the durable train floor allowed by adoption and bootstrap evidence.""" + adoption_floor = DURABLE_MIGRATION_ADOPTION_FLOORS[tier] + return max(adoption_floor, bootstrap_versions.get(tier, adoption_floor)) + + def _prove_and_release_persisted_train( archive_root: Path, manifest_path: Path, @@ -1923,6 +1941,16 @@ def execute_durable_change_train( live, train, current_target_version=runtime_target_version, + actual_evidence=(forward_version_evidence[tier].actual if tier in forward_version_evidence else None), + integrity_check=( + forward_version_evidence[tier].integrity_check if tier in forward_version_evidence else None + ), + live_inventory=( + forward_version_evidence[tier].live_inventory if tier in forward_version_evidence else None + ), + canonical_inventory=( + forward_version_evidence[tier].canonical_inventory if tier in forward_version_evidence else None + ), ) return DurableChangeTrainExecution( train=train, @@ -2045,6 +2073,7 @@ def _reconcile_durable_change_train_startup_locked( live_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} canonical_inventory_by_tier: dict[ArchiveTier, _migration_runner.DurableSchemaInventory] = {} manifests_by_tier: dict[ArchiveTier, dict[int, DurableChangeTrain]] = {} + validated_tiers: set[ArchiveTier] = set() manifest_paths = tuple(sorted(manifest_root.glob("*.json"))) fresh_bootstrap_versions = _fresh_durable_bootstrap_versions(archive_root, manifest_root) @@ -2116,13 +2145,15 @@ def record_reconciled(path: Path) -> None: f"{tier.value} durable tier regressed below fresh bootstrap v{bootstrap_version}" ) if bootstrap_version == current_version and not tier_manifest_paths: + validated_tiers.add(tier) continue _require_released_train_chain( tier, manifests_by_tier[tier], current_version=current_version, - floor=max(adoption_floor, bootstrap_version or adoption_floor), + floor=_chain_floor(tier, fresh_bootstrap_versions), ) + validated_tiers.add(tier) for manifest_path in manifest_paths: train = load_durable_change_train_manifest(manifest_path) @@ -2133,17 +2164,17 @@ def record_reconciled(path: Path) -> None: if actual is None: actual = capture_durable_database_evidence(live, train.tier) live_evidence_by_tier[train.tier] = actual - if actual.user_version > DURABLE_MIGRATION_ADOPTION_FLOORS[train.tier]: + if ( + actual.user_version > DURABLE_MIGRATION_ADOPTION_FLOORS[train.tier] + and train.tier not in validated_tiers + ): if train.tier not in manifests_by_tier: manifests_by_tier[train.tier] = _released_train_manifests_by_target(manifest_root, train.tier) _require_released_train_chain( train.tier, manifests_by_tier[train.tier], current_version=actual.user_version, - floor=max( - DURABLE_MIGRATION_ADOPTION_FLOORS[train.tier], - fresh_bootstrap_versions.get(train.tier, DURABLE_MIGRATION_ADOPTION_FLOORS[train.tier]), - ), + floor=_chain_floor(train.tier, fresh_bootstrap_versions), ) if actual.user_version > train.target_version: if train.tier not in live_integrity_by_tier: diff --git a/tests/unit/daemon/test_raw_failure_sample.py b/tests/unit/daemon/test_raw_failure_sample.py index 03e51c7a51..152058f84b 100644 --- a/tests/unit/daemon/test_raw_failure_sample.py +++ b/tests/unit/daemon/test_raw_failure_sample.py @@ -570,6 +570,67 @@ def test_daemon_status_lifecycle_counts_match_the_shared_projection(self, tmp_pa assert info["terminal_rejections"] == snapshot.terminal == 1 assert info["unexplained_failures"] == snapshot.unexplained == 0 + def test_daemon_status_uses_the_lifecycle_sample_rows(self, tmp_path: Path) -> None: + """Status must retain classified rows selected ahead of newer unexplained rows.""" + source_db = tmp_path / "source.db" + initialize_archive_database(source_db, ArchiveTier.SOURCE) + with sqlite3.connect(source_db) as conn: + conn.executemany( + """ + INSERT INTO raw_sessions ( + raw_id, origin, native_id, source_path, source_index, blob_hash, blob_size, + acquired_at_ms, parse_error + ) VALUES (?, ?, ?, ?, 0, ?, 0, ?, ?) + """, + [ + ( + f"raw-unexplained-{index}", + "codex-session", + f"native-{index}", + "/data/unexplained.jsonl", + bytes(index.to_bytes(32, "big")), + 1_770_000_000_100 + index, + "newer unexplained failure", + ) + for index in range(50) + ] + + [ + ( + "raw-terminal", + "unknown-export", + "terminal-native", + "/data/terminal.json", + bytes(32), + 1_770_000_000_000, + "parsed raw payload produced no sessions", + ) + ], + ) + upsert_raw_artifact( + conn, + "raw-terminal", + ArchiveSourceArtifact( + artifact_id="terminal-evidence", + origin="unknown-export", + source_path="/data/terminal.json", + source_index=0, + artifact_kind="terminal_unsupported_shape", + classification_reason="terminal_unsupported_shape", + support_status=ArtifactSupportStatus.UNSUPPORTED_PARSEABLE, + ), + ) + conn.commit() + + with ( + patch("polylogue.daemon.status.archive_root", return_value=tmp_path), + patch("polylogue.daemon.status._active_status_db_path", return_value=tmp_path / "index.db"), + ): + info = _raw_failure_info() + + samples = cast(list[RawFailureSample], info["samples"]) + assert len(samples) == 50 + assert any(sample.provider_hint == "unknown-export" and sample.lifecycle == "terminal" for sample in samples) + def test_lifecycle_samples_are_sql_bounded_for_large_file_backed_failure_sets( self, tmp_path: Path, @@ -763,7 +824,11 @@ def fetchall(self) -> list[object]: def __iter__(self) -> object: normalized = " ".join(self._sql.split()).lower() - if "from raw_sessions as r" in normalized and "limit 50" not in normalized: + if ( + "from raw_sessions as r" in normalized + and "limit 50" not in normalized + and "r.raw_id in (" not in normalized + ): raise AssertionError("raw-failure lifecycle totals must aggregate in SQL") return iter(self._cursor) diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 4b3eea12c8..679604104c 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1317,14 +1317,16 @@ def test_released_train_chain_is_anchored_at_adoption_floor() -> None: def test_released_train_chain_can_start_at_bootstrap_floor(monkeypatch: pytest.MonkeyPatch) -> None: + floor = DURABLE_MIGRATION_ADOPTION_FLOORS[ArchiveTier.SOURCE] + current_version = floor + 1 released = cast(DurableChangeTrain, SimpleNamespace(state=DurableChangeTrainState.RELEASED)) monkeypatch.setattr(durable_change_train_module, "_historical_schema_evidence", lambda _train: None) durable_change_train_module._require_released_train_chain( ArchiveTier.SOURCE, - {30: released}, - current_version=30, - floor=29, + {current_version: released}, + current_version=current_version, + floor=floor, ) @@ -1538,6 +1540,19 @@ def test_fresh_bootstrap_receipt_rejects_archive_identity_mismatch(tmp_path: Pat reconcile_durable_change_train_startup(tmp_path) +def test_fresh_bootstrap_receipt_rejects_recorded_version_tampering(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + marker = tmp_path / ".maintenance-state" / "durable-change-trains" / ".bootstrap" + payload = json.loads(marker.read_text(encoding="utf-8")) + cast(dict[str, int], payload["versions"])["source"] += 1 + marker.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(DurableChangeTrainError, match="marker digest mismatch"): + reconcile_durable_change_train_startup(tmp_path) + + def test_source_train_identity_survives_late_user_tier_initialization(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_archive_database From 5b324a889fad32da5b03c7e934ecd8544ccf49af Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 00:32:01 +0200 Subject: [PATCH 30/32] fix(storage): publish bootstrap marker atomically Problem Fresh durable bootstrap markers were written directly to their final path, and audit-tier no-op maintenance could enter a source/user-only train-floor lookup. What changed Publish the marker through a flushed temporary file and directory fsync. Skip forward-version train-floor validation for the audit tier, which has no durable change-train adoption floor, and cover that route with a focused test. Verification Ref #3875 Passed: direnv exec . devtools test tests/unit/daemon/test_raw_failure_sample.py tests/unit/storage/test_durable_change_train.py tests/unit/cli/test_archive_maintenance_cli.py::test_migrate_tier_cli_exposes_forward_version_receipt Result: 95 passed. Co-Authored-By: Claude --- .../storage/sqlite/durable_change_train.py | 26 +++++++++++++++---- .../unit/storage/test_durable_change_train.py | 14 ++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index dccb956dea..0919c195f4 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -339,10 +339,26 @@ def _record_fresh_durable_bootstrap(archive_root: Path) -> None: "versions": versions, } payload["marker_digest"] = _bootstrap_marker_digest(payload) - marker_root.joinpath(_FRESH_DURABLE_BOOTSTRAP_MARKER).write_text( - json.dumps(payload, sort_keys=True, separators=(",", ":")), - encoding="utf-8", - ) + marker_path = marker_root / _FRESH_DURABLE_BOOTSTRAP_MARKER + encoded = (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile( + dir=marker_root, + prefix=f".{_FRESH_DURABLE_BOOTSTRAP_MARKER}.", + suffix=".tmp", + delete=False, + ) as stream: + temporary = Path(stream.name) + stream.write(encoded) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, marker_path) + temporary = None + _migration_runner._fsync_manifest_directory(marker_root) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) def _fresh_durable_bootstrap_versions(archive_root: Path, marker_root: Path) -> dict[ArchiveTier, int]: @@ -1703,7 +1719,7 @@ def _forward_version_receipt_for_current_tier( for train in manifests_by_target.values() if train.state is DurableChangeTrainState.RELEASED and train.target_version < current_version ] - if current_version > DURABLE_MIGRATION_ADOPTION_FLOORS[tier]: + if tier in DURABLE_MIGRATION_ADOPTION_FLOORS and current_version > DURABLE_MIGRATION_ADOPTION_FLOORS[tier]: _require_released_train_chain( tier, manifests_by_target, diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 679604104c..b466b38052 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1362,6 +1362,20 @@ def test_forward_receipt_checks_missing_chain_before_empty_history( ) +def test_forward_receipt_skips_non_train_audit_tier(tmp_path: Path) -> None: + with sqlite3.connect(":memory:") as conn: + assert ( + durable_change_train_module._forward_version_receipt_for_current_tier( + tmp_path, + conn, + ArchiveTier.AUDIT, + current_version=1, + current_target_version=1, + ) + is None + ) + + def test_startup_recovers_later_train_before_released_chain_validation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 424c773f04ea58818fa051e9a0731c53c7aef5b8 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 00:37:07 +0200 Subject: [PATCH 31/32] fix(storage): fail closed on incomplete pre-marker archives Problem Pre-marker adoption inferred legacy provenance from missing train state and could recreate a missing user tier before recording a new baseline. What changed Require the pre-existing train directory and every durable tier before adoption. Missing train state or a missing user tier now follows the fail-closed chain path and leaves the archive untouched. Add regressions for both loss cases. Verification Ref #3875 Passed: direnv exec . devtools test tests/unit/storage/test_durable_change_train.py tests/unit/daemon/test_raw_failure_sample.py tests/unit/cli/test_archive_maintenance_cli.py::test_migrate_tier_cli_exposes_forward_version_receipt Result: 97 passed. Co-Authored-By: Claude --- .../storage/sqlite/archive_tiers/bootstrap.py | 3 ++- .../unit/storage/test_durable_change_train.py | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/polylogue/storage/sqlite/archive_tiers/bootstrap.py b/polylogue/storage/sqlite/archive_tiers/bootstrap.py index 49d5c71cee..7c42aa1bc9 100644 --- a/polylogue/storage/sqlite/archive_tiers/bootstrap.py +++ b/polylogue/storage/sqlite/archive_tiers/bootstrap.py @@ -329,7 +329,8 @@ def initialize_active_archive_root(root: Path) -> None: fresh_durable_bootstrap = not durable_tier_exists and not has_durable_train_state and not has_bootstrap_marker pre_marker_adoption = ( (root / archive_tier_spec(ArchiveTier.SOURCE).filename).is_file() - and durable_tier_exists + and all((root / archive_tier_spec(tier).filename).is_file() for tier in DURABLE_MIGRATION_TIERS) + and manifest_root.is_dir() and not has_durable_train_state and not has_bootstrap_marker ) diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index b466b38052..d175e8bde3 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1532,6 +1532,30 @@ def test_pre_marker_current_archive_is_adopted_once(tmp_path: Path) -> None: assert marker.is_file() +def test_pre_marker_adoption_refuses_missing_train_directory(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + marker_root = tmp_path / ".maintenance-state" / "durable-change-trains" + (marker_root / ".bootstrap").unlink() + marker_root.rmdir() + + with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): + initialize_active_archive_root(tmp_path) + + +def test_pre_marker_adoption_requires_all_durable_tiers(tmp_path: Path) -> None: + from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root + + initialize_active_archive_root(tmp_path) + (tmp_path / ".maintenance-state" / "durable-change-trains" / ".bootstrap").unlink() + (tmp_path / "user.db").unlink() + + with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): + initialize_active_archive_root(tmp_path) + assert not (tmp_path / "user.db").exists() + + def test_bootstrap_marker_survives_index_generation_replacement(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root From 06e9cdbe6d09582b936839954ea660f43511186b Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 8 Aug 2026 00:57:32 +0200 Subject: [PATCH 32/32] fix(storage): recover interrupted fresh bootstrap Problem: A failure while creating a later archive tier could leave source.db and other fresh files without durable bootstrap evidence. The next startup treated that partial fresh archive as an established archive with missing train authority.\n\nWhat changed: Persist an authenticated root-bound bootstrap intent before creating tier files. Startup validates and resumes an incomplete fresh bootstrap, publishes the completed marker atomically, and fails closed on tampered intents or conflicting train state. Add recovery and tamper regressions.\n\nCompatibility/migration: Existing completed bootstrap markers and pre-marker adoption remain unchanged. --- .../storage/sqlite/archive_tiers/bootstrap.py | 38 ++++++-- .../storage/sqlite/durable_change_train.py | 89 ++++++++++++++++++- .../unit/storage/test_durable_change_train.py | 71 +++++++++++++++ 3 files changed, 191 insertions(+), 7 deletions(-) diff --git a/polylogue/storage/sqlite/archive_tiers/bootstrap.py b/polylogue/storage/sqlite/archive_tiers/bootstrap.py index 7c42aa1bc9..c7f78bfe7a 100644 --- a/polylogue/storage/sqlite/archive_tiers/bootstrap.py +++ b/polylogue/storage/sqlite/archive_tiers/bootstrap.py @@ -312,6 +312,11 @@ def initialize_archive_database( def initialize_active_archive_root(root: Path) -> None: """Create or initialize every tier database in an archive root.""" from polylogue.storage.archive_identity import ArchiveLocation, OwnedArchiveLocation + from polylogue.storage.sqlite.durable_change_train import ( + _record_fresh_durable_bootstrap, + _record_fresh_durable_bootstrap_intent, + _validate_fresh_durable_bootstrap_intent, + ) with OwnedArchiveLocation.acquire( ArchiveLocation.resolve(root), @@ -326,27 +331,50 @@ def initialize_active_archive_root(root: Path) -> None: manifest_root = root / ".maintenance-state" / "durable-change-trains" has_durable_train_state = any(manifest_root.glob("*.json")) has_bootstrap_marker = (manifest_root / ".bootstrap").is_file() - fresh_durable_bootstrap = not durable_tier_exists and not has_durable_train_state and not has_bootstrap_marker + pending_bootstrap_path = manifest_root / ".bootstrap.pending" + has_pending_bootstrap = pending_bootstrap_path.is_file() + if has_pending_bootstrap: + _validate_fresh_durable_bootstrap_intent(root) + if has_durable_train_state: + raise RuntimeError( + "fresh durable bootstrap intent conflicts with durable train state; " + "refusing to guess which authority is current" + ) + fresh_durable_bootstrap = ( + not durable_tier_exists + and not has_durable_train_state + and not has_bootstrap_marker + and not has_pending_bootstrap + ) + recovering_fresh_durable_bootstrap = fresh_durable_bootstrap or ( + has_pending_bootstrap and not has_bootstrap_marker + ) pre_marker_adoption = ( (root / archive_tier_spec(ArchiveTier.SOURCE).filename).is_file() and all((root / archive_tier_spec(tier).filename).is_file() for tier in DURABLE_MIGRATION_TIERS) and manifest_root.is_dir() and not has_durable_train_state and not has_bootstrap_marker + and not has_pending_bootstrap ) - if not fresh_durable_bootstrap and not pre_marker_adoption: + if fresh_durable_bootstrap: + _record_fresh_durable_bootstrap_intent(root) + if not recovering_fresh_durable_bootstrap and not pre_marker_adoption: reconcile_durable_change_trains_on_startup(root) for spec in ARCHIVE_TIER_SPECS.values(): initialize_archive_database(root / spec.filename, spec.tier) - if fresh_durable_bootstrap: - from polylogue.storage.sqlite.durable_change_train import _record_fresh_durable_bootstrap - + if recovering_fresh_durable_bootstrap: _record_fresh_durable_bootstrap(root) elif pre_marker_adoption: from polylogue.storage.sqlite.durable_change_train import _adopt_pre_marker_durable_bootstrap _adopt_pre_marker_durable_bootstrap(root) reconcile_durable_change_trains_on_startup(root) + elif has_pending_bootstrap: + # A crash after publishing the completed marker but before + # removing the intent is harmless. Keep the intent until the + # completed marker has passed normal startup reconciliation. + pending_bootstrap_path.unlink(missing_ok=True) def reconcile_durable_change_trains_on_startup(root: Path) -> tuple[Path, ...]: diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 0919c195f4..64b0f60731 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -76,6 +76,7 @@ _SOURCE_CONTINUITY_PENDING_FORMAT = "polylogue.source-continuity-pending.v1" _FRESH_DURABLE_BOOTSTRAP_FORMAT = "polylogue.durable-bootstrap.v1" _FRESH_DURABLE_BOOTSTRAP_MARKER = ".bootstrap" +_FRESH_DURABLE_BOOTSTRAP_PENDING_MARKER = ".bootstrap.pending" class DurableSourceTrainMissingError(DurableChangeTrainError): @@ -326,8 +327,12 @@ def _record_fresh_durable_bootstrap(archive_root: Path) -> None: archive_root = archive_root.resolve() marker_root = archive_root / ".maintenance-state" / "durable-change-trains" - if (marker_root / _FRESH_DURABLE_BOOTSTRAP_MARKER).exists() or any(marker_root.glob("*.json")): + marker_path = marker_root / _FRESH_DURABLE_BOOTSTRAP_MARKER + pending_path = marker_root / _FRESH_DURABLE_BOOTSTRAP_PENDING_MARKER + if marker_path.exists() or any(marker_root.glob("*.json")): raise DurableChangeTrainError(f"cannot record fresh durable bootstrap over existing train state: {marker_root}") + if pending_path.is_file(): + _validate_fresh_durable_bootstrap_intent(archive_root) marker_root.mkdir(parents=True, exist_ok=True) versions: dict[str, int] = {} for tier in DURABLE_MIGRATION_ADOPTION_FLOORS: @@ -339,13 +344,83 @@ def _record_fresh_durable_bootstrap(archive_root: Path) -> None: "versions": versions, } payload["marker_digest"] = _bootstrap_marker_digest(payload) + _write_bootstrap_receipt(marker_path, payload) + pending_path.unlink(missing_ok=True) + + +def _record_fresh_durable_bootstrap_intent(archive_root: Path) -> None: + """Record an authenticated intent before creating the first tier file. + + Fresh archive initialization creates several independent SQLite files. A + failure in a later tier can therefore leave a partial fresh archive. The + intent distinguishes that recoverable state from an established archive + whose durable train evidence has been lost. + """ + from polylogue.storage.sqlite.archive_tiers import ARCHIVE_VERSION_BY_TIER + + archive_root = archive_root.resolve() + marker_root = archive_root / ".maintenance-state" / "durable-change-trains" marker_path = marker_root / _FRESH_DURABLE_BOOTSTRAP_MARKER + pending_path = marker_root / _FRESH_DURABLE_BOOTSTRAP_PENDING_MARKER + if marker_path.exists() or any(marker_root.glob("*.json")): + raise DurableChangeTrainError( + f"cannot record fresh durable bootstrap intent over existing train state: {marker_root}" + ) + if pending_path.is_file(): + _validate_fresh_durable_bootstrap_intent(archive_root) + return + versions = {tier.value: ARCHIVE_VERSION_BY_TIER[tier] for tier in DURABLE_MIGRATION_ADOPTION_FLOORS} + payload: dict[str, object] = { + "format": _FRESH_DURABLE_BOOTSTRAP_FORMAT, + "state": "pending", + "durable_identity_digest": _fresh_bootstrap_intent_identity_digest(archive_root), + "versions": versions, + } + payload["marker_digest"] = _bootstrap_marker_digest(payload) + _write_bootstrap_receipt(pending_path, payload) + + +def _validate_fresh_durable_bootstrap_intent(archive_root: Path) -> None: + """Validate the authenticated intent for a recoverable fresh bootstrap.""" + from polylogue.storage.sqlite.archive_tiers import ARCHIVE_VERSION_BY_TIER + + archive_root = archive_root.resolve() + marker_path = ( + archive_root / ".maintenance-state" / "durable-change-trains" / _FRESH_DURABLE_BOOTSTRAP_PENDING_MARKER + ) + try: + payload = json.loads(marker_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise DurableChangeTrainError(f"invalid fresh durable bootstrap intent: {marker_path}") from exc + if not isinstance(payload, dict) or payload.get("format") != _FRESH_DURABLE_BOOTSTRAP_FORMAT: + raise DurableChangeTrainError(f"fresh durable bootstrap intent format mismatch: {marker_path}") + if payload.get("state") != "pending": + raise DurableChangeTrainError(f"fresh durable bootstrap intent state is invalid: {marker_path}") + if payload.get("durable_identity_digest") != _fresh_bootstrap_intent_identity_digest(archive_root): + raise DurableChangeTrainError("fresh durable bootstrap intent durable identity mismatch") + marker_digest = payload.get("marker_digest") + unsigned_payload = dict(payload) + unsigned_payload.pop("marker_digest", None) + if not isinstance(marker_digest, str) or marker_digest != _bootstrap_marker_digest(unsigned_payload): + raise DurableChangeTrainError("fresh durable bootstrap intent digest mismatch") + raw_versions = payload.get("versions") + if not isinstance(raw_versions, dict): + raise DurableChangeTrainError(f"fresh durable bootstrap intent versions are invalid: {marker_path}") + for tier in DURABLE_MIGRATION_ADOPTION_FLOORS: + if raw_versions.get(tier.value) != ARCHIVE_VERSION_BY_TIER[tier]: + raise DurableChangeTrainError(f"fresh durable bootstrap intent target version is stale: {marker_path}") + + +def _write_bootstrap_receipt(marker_path: Path, payload: dict[str, object]) -> None: + """Atomically publish one bootstrap receipt and fsync its directory.""" + marker_root = marker_path.parent + marker_root.mkdir(parents=True, exist_ok=True) encoded = (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") temporary: Path | None = None try: with tempfile.NamedTemporaryFile( dir=marker_root, - prefix=f".{_FRESH_DURABLE_BOOTSTRAP_MARKER}.", + prefix=f".{marker_path.name}.", suffix=".tmp", delete=False, ) as stream: @@ -361,6 +436,16 @@ def _record_fresh_durable_bootstrap(archive_root: Path) -> None: temporary.unlink(missing_ok=True) +def _fresh_bootstrap_intent_identity_digest(archive_root: Path) -> str: + """Bind a pre-file bootstrap intent to its root without inode identity.""" + return _canonical_json_sha256( + { + "configured_root": str(archive_root.resolve().absolute()), + "purpose": "fresh-durable-bootstrap", + } + ) + + def _fresh_durable_bootstrap_versions(archive_root: Path, marker_root: Path) -> dict[ArchiveTier, int]: """Return direct-bootstrap versions when the marker is authentic.""" from polylogue.storage.archive_identity import ArchiveIdentity diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index d175e8bde3..affde4465f 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -1521,6 +1521,77 @@ def test_fresh_archive_bootstrap_receipt_allows_repeat_startup(tmp_path: Path) - initialize_active_archive_root(tmp_path) +def test_fresh_bootstrap_intent_recovers_after_late_tier_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from polylogue.storage.sqlite.archive_tiers import bootstrap + + real_initialize_archive_database = bootstrap.initialize_archive_database + failed = False + + def fail_embeddings_once( + path: Path, + tier: ArchiveTier, + *, + allow_create: bool = True, + expected_version: int | None = None, + ) -> None: + nonlocal failed + if tier is ArchiveTier.EMBEDDINGS and not failed: + failed = True + raise RuntimeError("simulated late fresh-bootstrap failure") + real_initialize_archive_database(path, tier, allow_create=allow_create, expected_version=expected_version) + + monkeypatch.setattr(bootstrap, "initialize_archive_database", fail_embeddings_once) + with pytest.raises(RuntimeError, match="simulated late fresh-bootstrap failure"): + bootstrap.initialize_active_archive_root(tmp_path) + + marker_root = tmp_path / ".maintenance-state" / "durable-change-trains" + assert (marker_root / ".bootstrap.pending").is_file() + assert not (marker_root / ".bootstrap").exists() + assert (tmp_path / "source.db").is_file() + + monkeypatch.setattr(bootstrap, "initialize_archive_database", real_initialize_archive_database) + bootstrap.initialize_active_archive_root(tmp_path) + + assert (marker_root / ".bootstrap").is_file() + assert not (marker_root / ".bootstrap.pending").exists() + assert reconcile_durable_change_train_startup(tmp_path) == () + + +def test_fresh_bootstrap_intent_rejects_tampering_before_recovery( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from polylogue.storage.sqlite.archive_tiers import bootstrap + + real_initialize_archive_database = bootstrap.initialize_archive_database + + def fail_embeddings( + path: Path, + tier: ArchiveTier, + *, + allow_create: bool = True, + expected_version: int | None = None, + ) -> None: + if tier is ArchiveTier.EMBEDDINGS: + raise RuntimeError("simulated late fresh-bootstrap failure") + real_initialize_archive_database(path, tier, allow_create=allow_create, expected_version=expected_version) + + monkeypatch.setattr(bootstrap, "initialize_archive_database", fail_embeddings) + with pytest.raises(RuntimeError, match="simulated late fresh-bootstrap failure"): + bootstrap.initialize_active_archive_root(tmp_path) + + pending = tmp_path / ".maintenance-state" / "durable-change-trains" / ".bootstrap.pending" + payload = json.loads(pending.read_text(encoding="utf-8")) + payload["durable_identity_digest"] = "0" * 64 + pending.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(DurableChangeTrainError, match="intent durable identity mismatch"): + bootstrap.initialize_active_archive_root(tmp_path) + + def test_pre_marker_current_archive_is_adopted_once(tmp_path: Path) -> None: from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root