From 4b4055ac9f76e72bda2ba0f047b4ed7f40bc686c Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:01:10 +0200 Subject: [PATCH 01/27] feat: add guarded raw-authority recovery actuators --- devtools/validation_lane_catalog_contracts.py | 9 + docs/maintenance.md | 22 + docs/plans/mutation-census.yaml | 16 + docs/test-quality-workflows.md | 10 +- polylogue/artifacts/runtime.py | 52 + .../cli/commands/maintenance/__init__.py | 6 + .../maintenance/_raw_authority_recovery.py | 96 ++ .../maintenance/raw_authority_recovery.py | 956 ++++++++++++++++++ polylogue/maintenance/raw_authority_reset.py | 62 +- polylogue/operations/specs.py | 46 + polylogue/storage/raw_authority.py | 36 +- .../unit/cli/test_maintenance_registration.py | 6 + .../maintenance/test_raw_authority_reset.py | 229 +++-- tests/unit/operations/test_mutation_census.py | 16 + tests/unit/operations/test_specs.py | 2 + 15 files changed, 1453 insertions(+), 111 deletions(-) create mode 100644 polylogue/cli/commands/maintenance/_raw_authority_recovery.py create mode 100644 polylogue/maintenance/raw_authority_recovery.py diff --git a/devtools/validation_lane_catalog_contracts.py b/devtools/validation_lane_catalog_contracts.py index 38dd4063d1..1897f627da 100644 --- a/devtools/validation_lane_catalog_contracts.py +++ b/devtools/validation_lane_catalog_contracts.py @@ -211,6 +211,7 @@ "tests/unit/operations/test_mutations.py", "tests/unit/operations/test_mutation_actuators.py", "tests/unit/operations/test_operation_bindings.py", + "tests/unit/maintenance/test_raw_authority_reset.py", "tests/unit/annotations/test_importer.py::test_import_roundtrip_keeps_failures_candidates_and_independent_batches", "tests/unit/cli/test_excise.py::TestExciseStandalone::test_yes_applies_excision", ), @@ -222,6 +223,7 @@ "blackboard-post-loop", "assertion-candidate-capture-loop", "raw-authority-blocker-resolution-loop", + "raw-authority-recovery-loop", "saved-view-mutation-loop", "recall-pack-mutation-loop", "workspace-mutation-loop", @@ -243,6 +245,11 @@ "raw_authority_plans", "raw_authority_blockers", "raw_authority_blocker_resolution", + "raw_authority_census_ledger", + "raw_authority_census_recovery_receipt", + "raw_revision_heads", + "raw_revision_applications", + "raw_authority_index_seed_recovery_receipt", ), operation_targets=( "mutate-add-tag", @@ -261,6 +268,8 @@ "mutate-update-index", "mutate-rebuild-insights", "mutate-resolve-raw-authority-blocker", + "mutate-reset-raw-authority-census", + "mutate-prune-orphaned-index-revision-seeds", "mutate-save-saved-view", "mutate-delete-saved-view", "mutate-save-recall-pack", diff --git a/docs/maintenance.md b/docs/maintenance.md index 00dbf9cee3..1ea7f11601 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -798,6 +798,28 @@ confirm-flag-strength authorization bound to that plan's hash, refusing (`preview_stale`) if the blocker was concurrently resolved between preview and confirm. +### `polylogue ops maintenance raw-authority-recovery` - break-glass ledger recovery + +This command family is the only operator route for the two callerless raw-authority recovery actuators. It is inspect-only by default. The census reset removes only the five poisoned census-planning tables after a verified source-tier backup. The index-seed prune removes only active-index `raw_revision_heads` and `raw_revision_applications` rows whose source raw is absent. Parser census rows, source raws, blob receipts, and present-source revision rows are outside both target sets. + +Write an exact plan first, then apply that same plan with the required backup authority: + +```bash +polylogue ops maintenance raw-authority-recovery \ + --operation reset_raw_authority_census \ + --plan-file /realm/tmp/work/raw-authority-census-reset.plan.json \ + --backup-manifest /realm/staging/polylogue-backup/manifest.json \ + --output-format json + +polylogue ops maintenance raw-authority-recovery \ + --operation reset_raw_authority_census --apply \ + --plan-file /realm/tmp/work/raw-authority-census-reset.plan.json \ + --backup-manifest /realm/staging/polylogue-backup/manifest.json \ + --output-format json +``` + +Apply refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation, runs one failure-atomic transaction, and writes a self-hashed immutable receipt. It does not invoke the broad index reset or reparse path. + ### Measuring Codex UUID-title coverage Codex sessions without a resolvable title (thread name / authored history / diff --git a/docs/plans/mutation-census.yaml b/docs/plans/mutation-census.yaml index 8a061d3428..a762421842 100644 --- a/docs/plans/mutation-census.yaml +++ b/docs/plans/mutation-census.yaml @@ -170,6 +170,22 @@ rows: adapters: - polylogue.cli.commands.maintenance._raw_identity.raw_authority_blocker_resolve_command + - operation: mutate-reset-raw-authority-census + spec_name: mutate-reset-raw-authority-census + status: executor-routed + actuator: polylogue.maintenance.raw_authority_recovery.ResetRawAuthorityCensusActuator + surfaces: [cli] + adapters: + - polylogue.cli.commands.maintenance._raw_authority_recovery.raw_authority_recovery_command + + - operation: mutate-prune-orphaned-index-revision-seeds + spec_name: mutate-prune-orphaned-index-revision-seeds + status: executor-routed + actuator: polylogue.maintenance.raw_authority_recovery.PruneOrphanedIndexRevisionSeedsActuator + surfaces: [cli] + adapters: + - polylogue.cli.commands.maintenance._raw_authority_recovery.raw_authority_recovery_command + # --- Phase 4: executor-routed (saved-view/recall-pack/workspace family) ---- - operation: mutate-save-saved-view diff --git a/docs/test-quality-workflows.md b/docs/test-quality-workflows.md index 1abe2124fb..0140b1efb5 100644 --- a/docs/test-quality-workflows.md +++ b/docs/test-quality-workflows.md @@ -24,11 +24,11 @@ Current registry snapshot: ## Runtime Coverage -- covered runtime paths: `37` -- covered runtime artifacts: `57` -- covered runtime operations: `55` +- covered runtime paths: `38` +- covered runtime artifacts: `62` +- covered runtime operations: `57` - covered maintenance targets: `5` -- covered declared operation targets: `77` +- covered declared operation targets: `79` - uncovered runtime paths: — - uncovered runtime artifacts: — - uncovered runtime operations: — @@ -399,7 +399,7 @@ These projections explain which executable lanes, inferred fixture scenarios, or | `validation-lane` | `maintenance-workflows` | — | — | — | — | — | — | `contract`
`maintenance` | Health, maintenance selection, cache/live provenance, and machine output | | `validation-lane` | `memory-budget` | `session-query-loop` | `message_fts`
`session_query_results` | — | — | `query-sessions` | — | `live`
`retrieval`
`readiness` | Live archive grouped retrieval command under an explicit RSS budget | | `validation-lane` | `mixed-consumer-contracts` | — | — | — | — | — | — | — | CLI, facade, and readiness surfaces consuming the same evidence/inference insight model | -| `validation-lane` | `mutation-routes` | `tag-mutation-loop`
`metadata-mutation-loop`
`mark-mutation-loop`
`annotation-mutation-loop`
`blackboard-post-loop`
`assertion-candidate-capture-loop`
`raw-authority-blocker-resolution-loop`
`saved-view-mutation-loop`
`recall-pack-mutation-loop`
`workspace-mutation-loop`
`correction-mutation-loop`
`session-delete-loop`
`session-excision-loop`
`identity-reset-loop`
`message-fts-readiness-loop`
`session-insight-repair-loop` | `sessions`
`assertions`
`archive_deleted_session`
`raw_sessions`
`blob_refs`
`excision_receipt`
`suppression_rows`
`raw_authority_plans`
`raw_authority_blockers`
`raw_authority_blocker_resolution` | — | — | `mutate-add-tag`
`mutate-remove-tag`
`mutate-bulk-tag-sessions`
`mutate-set-metadata`
`mutate-delete-metadata`
`mutate-add-mark`
`mutate-remove-mark`
`mutate-save-annotation`
`mutate-delete-annotation`
`mutate-blackboard-post`
`mutate-capture-assertion-candidate`
`mutate-import-annotation-batch`
`mutate-rebuild-index`
`mutate-update-index`
`mutate-rebuild-insights`
`mutate-resolve-raw-authority-blocker`
`mutate-save-saved-view`
`mutate-delete-saved-view`
`mutate-save-recall-pack`
`mutate-delete-recall-pack`
`mutate-save-workspace`
`mutate-delete-workspace`
`mutate-record-correction`
`mutate-delete-correction`
`mutate-clear-corrections`
`mutate-delete-session`
`mutate-session-excision`
`mutate-identity-reset` | — | `contract`
`mutation`
`operation-executor` | Executor-routed mutation actuators and transaction receipts over their declared runtime closures | +| `validation-lane` | `mutation-routes` | `tag-mutation-loop`
`metadata-mutation-loop`
`mark-mutation-loop`
`annotation-mutation-loop`
`blackboard-post-loop`
`assertion-candidate-capture-loop`
`raw-authority-blocker-resolution-loop`
`raw-authority-recovery-loop`
`saved-view-mutation-loop`
`recall-pack-mutation-loop`
`workspace-mutation-loop`
`correction-mutation-loop`
`session-delete-loop`
`session-excision-loop`
`identity-reset-loop`
`message-fts-readiness-loop`
`session-insight-repair-loop` | `sessions`
`assertions`
`archive_deleted_session`
`raw_sessions`
`blob_refs`
`excision_receipt`
`suppression_rows`
`raw_authority_plans`
`raw_authority_blockers`
`raw_authority_blocker_resolution`
`raw_authority_census_ledger`
`raw_authority_census_recovery_receipt`
`raw_revision_heads`
`raw_revision_applications`
`raw_authority_index_seed_recovery_receipt` | — | — | `mutate-add-tag`
`mutate-remove-tag`
`mutate-bulk-tag-sessions`
`mutate-set-metadata`
`mutate-delete-metadata`
`mutate-add-mark`
`mutate-remove-mark`
`mutate-save-annotation`
`mutate-delete-annotation`
`mutate-blackboard-post`
`mutate-capture-assertion-candidate`
`mutate-import-annotation-batch`
`mutate-rebuild-index`
`mutate-update-index`
`mutate-rebuild-insights`
`mutate-resolve-raw-authority-blocker`
`mutate-reset-raw-authority-census`
`mutate-prune-orphaned-index-revision-seeds`
`mutate-save-saved-view`
`mutate-delete-saved-view`
`mutate-save-recall-pack`
`mutate-delete-recall-pack`
`mutate-save-workspace`
`mutate-delete-workspace`
`mutate-record-correction`
`mutate-delete-correction`
`mutate-clear-corrections`
`mutate-delete-session`
`mutate-session-excision`
`mutate-identity-reset` | — | `contract`
`mutation`
`operation-executor` | Executor-routed mutation actuators and transaction receipts over their declared runtime closures | | `validation-lane` | `pipeline-probe-chatgpt` | `source-acquisition-loop`
`raw-reparse-loop`
`raw-archive-ingest-loop` | `configured_sources`
`source_payload_stream`
`raw_validation_state`
`artifact_observation_rows`
`validation_backlog`
`parse_backlog`
`parse_quarantine`
`archive_session_rows` | — | — | `acquire-raw-sessions`
`plan-validation-backlog`
`plan-parse-backlog`
`ingest-archive-runtime` | — | — | Synthetic ChatGPT parse-stage pipeline probe under explicit runtime and RSS budgets | | `validation-lane` | `probabilistic-enrichment-cleanup-live` | `archive-debt-query-loop`
`message-fts-readiness-loop`
`retrieval-band-readiness-loop` | `archive_readiness`
`embedding_status_results`
`message_fts`
`archive_debt_results`
`session_insight_readiness`
`retrieval_band_readiness` | — | — | `query-archive-debt`
`cli.json-contract`
`project-archive-readiness` | — | `insights`
`debt`
`live`
`maintenance`
`preview` | Bounded live archive lane for cleanup/debt preview and maintenance budgets | | `validation-lane` | `probabilistic-enrichment-contracts` | — | — | — | — | — | — | — | Session-enrichment contracts across CLI, facade, storage, and retrieval-band status | diff --git a/polylogue/artifacts/runtime.py b/polylogue/artifacts/runtime.py index c99cd53360..a385270d83 100644 --- a/polylogue/artifacts/runtime.py +++ b/polylogue/artifacts/runtime.py @@ -635,6 +635,46 @@ code_refs=("polylogue.operations.mutation_actuators.BlockerResolveActuator",), readiness_surfaces=("cli", "maintenance"), ), + ArtifactNode( + name="raw_authority_census_ledger", + layer=ArtifactLayer.DURABLE, + description="Poisonable source-tier census-planning bookkeeping cleared only by guarded recovery.", + depends_on=("raw_sessions",), + code_refs=("polylogue.maintenance.raw_authority_recovery",), + readiness_surfaces=("cli", "maintenance"), + ), + ArtifactNode( + name="raw_authority_census_recovery_receipt", + layer=ArtifactLayer.DURABLE, + description="Immutable receipt for a guarded raw-authority census-ledger recovery.", + depends_on=("raw_authority_census_ledger",), + code_refs=("polylogue.maintenance.raw_authority_recovery",), + readiness_surfaces=("cli", "maintenance"), + ), + ArtifactNode( + name="raw_revision_heads", + layer=ArtifactLayer.PROJECTION, + description="Active-index raw revision frontier heads used by orphan-seed recovery.", + depends_on=("raw_sessions",), + code_refs=("polylogue.storage.sqlite.archive_tiers.index",), + readiness_surfaces=("cli", "maintenance"), + ), + ArtifactNode( + name="raw_revision_applications", + layer=ArtifactLayer.PROJECTION, + description="Active-index raw revision decisions used by orphan-seed recovery.", + depends_on=("raw_sessions",), + code_refs=("polylogue.storage.sqlite.archive_tiers.index",), + readiness_surfaces=("cli", "maintenance"), + ), + ArtifactNode( + name="raw_authority_index_seed_recovery_receipt", + layer=ArtifactLayer.PROJECTION, + description="Immutable receipt for a guarded active-index orphan-seed prune.", + depends_on=("raw_revision_heads", "raw_revision_applications"), + code_refs=("polylogue.maintenance.raw_authority_recovery",), + readiness_surfaces=("cli", "maintenance"), + ), ) RUNTIME_ARTIFACT_PATHS: tuple[ArtifactPath, ...] = ( @@ -920,6 +960,18 @@ description="Raw evidence plans and blockers through their durable operator resolution receipt.", nodes=("raw_sessions", "raw_authority_plans", "raw_authority_blockers", "raw_authority_blocker_resolution"), ), + ArtifactPath( + name="raw-authority-recovery-loop", + description="Guarded source census reset and active-index orphan-seed recovery with immutable receipts.", + nodes=( + "raw_sessions", + "raw_authority_census_ledger", + "raw_authority_census_recovery_receipt", + "raw_revision_heads", + "raw_revision_applications", + "raw_authority_index_seed_recovery_receipt", + ), + ), ArtifactPath( name="saved-view-mutation-loop", description="Session query context through durable saved-view assertion mutation.", diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index 6513730dbf..a09b7eb6e2 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -101,6 +101,12 @@ "raw_authority_blocker_resolve_command", "Resolve one stale-plan blocker against current source evidence.", ), + ( + "raw-authority-recovery", + "_raw_authority_recovery", + "raw_authority_recovery_command", + "Inspect or apply one exact guarded raw-authority recovery plan.", + ), ("preview", "_preview", "preview_command", "Staleness inventory by model and scope. Read-only."), ("blob-gc", "_blob_gc", "blob_gc_command", "Preview lease-safe blob garbage collection. Read-only."), ( diff --git a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py new file mode 100644 index 0000000000..a80d1ecb49 --- /dev/null +++ b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py @@ -0,0 +1,96 @@ +"""CLI adapter for the typed raw-authority recovery family.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import click + +from polylogue.cli.shared.types import AppEnv +from polylogue.maintenance.raw_authority_recovery import ( + RawAuthorityRecoveryError, + RawAuthorityRecoveryPlan, + RecoveryOperation, + apply_raw_authority_recovery, + inspect_raw_authority_recovery, + write_recovery_plan, +) + + +@click.command("raw-authority-recovery") +@click.option( + "--operation", + type=click.Choice([operation.value for operation in RecoveryOperation]), + required=True, + help="One exact recovery actuator to inspect or apply.", +) +@click.option("--apply", "apply_changes", is_flag=True, help="Apply the exact previously inspected plan.") +@click.option( + "--plan-file", + type=click.Path(dir_okay=False, path_type=Path), + default=None, + help="Plan artifact to write during inspect or consume during --apply.", +) +@click.option( + "--backup-manifest", + type=click.Path(dir_okay=False, path_type=Path), + default=None, + help="Verified source/index backup authority, required for apply.", +) +@click.option( + "--receipt-file", + type=click.Path(dir_okay=False, path_type=Path), + default=None, + help="Immutable receipt destination bound into the dry-run plan.", +) +@click.option("--output-format", type=click.Choice(["plain", "json"]), default="plain", show_default=True) +@click.pass_obj +def raw_authority_recovery_command( + env: AppEnv, + operation: str, + apply_changes: bool, + plan_file: Path | None, + backup_manifest: Path | None, + receipt_file: Path | None, + output_format: str, +) -> None: + """Inspect raw-authority recovery, or apply one exact guarded plan.""" + plan_obj: RawAuthorityRecoveryPlan | None = None + try: + selected = RecoveryOperation(operation) + if apply_changes: + if plan_file is None: + raise click.ClickException("--apply requires --plan-file from a prior inspect") + report = apply_raw_authority_recovery(plan_file, backup_manifest=backup_manifest) + payload = report.to_dict() + else: + plan_obj = inspect_raw_authority_recovery( + env.config.archive_root, + selected, + backup_manifest=backup_manifest, + receipt_path=receipt_file, + ) + if plan_file is not None: + write_recovery_plan(plan_obj, plan_file) + payload = {"mode": "dry-run", **plan_obj.to_dict()} + except (RawAuthorityRecoveryError, FileNotFoundError, OSError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + if output_format == "json": + click.echo(json.dumps(payload, indent=2, sort_keys=True)) + return + if apply_changes: + click.echo(f"Raw-authority recovery: {payload.get('status', 'unknown')}") + click.echo(f"Operation: {payload.get('operation_id', 'unknown')}") + if payload.get("receipt_path"): + click.echo(f"Receipt: {payload['receipt_path']}") + return + assert plan_obj is not None + click.echo("Raw-authority recovery dry-run") + click.echo(f"Operation: {plan_obj.operation_id}") + click.echo(f"Plan digest: {plan_obj.plan_digest}") + click.echo(f"Before: {json.dumps(plan_obj.before_counts, sort_keys=True)}") + click.echo(f"Plan file: {plan_file if plan_file is not None else 'stdout only'}") + + +__all__ = ["raw_authority_recovery_command"] diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py new file mode 100644 index 0000000000..a1a0ad9207 --- /dev/null +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -0,0 +1,956 @@ +"""Guarded, receipted recovery for the raw-authority derived state. + +This is the only production mutation route for the two break-glass repairs in +this module. The storage helpers expose counts for diagnostics, but they do +not authorize deletion. A plan is an exact snapshot of one archive and one +operation. APPLY rechecks that snapshot while holding both archive ownership +and the rebuild lease, then performs one transaction against one tier. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sqlite3 +import tempfile +import uuid +from collections.abc import Mapping +from contextlib import closing +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import cast + +from polylogue.config import Config +from polylogue.maintenance.offline_guard import offline_maintenance_block_reason, running_daemon_pid +from polylogue.operations.mutation_transaction import ( + ConfirmationStrength, + DestructiveClass, + MutationPlan, + MutationReceipt, + MutationTransactionError, + OperationExecutor, + PlanStaleError, + build_plan, + make_target_ref, +) +from polylogue.paths import render_root +from polylogue.storage.archive_identity import ( + ArchiveLocation, + ArchiveLocationError, + ArchiveOwnershipError, + OwnedArchiveLocation, + assert_owns_archive_location, +) +from polylogue.storage.index_generation import RebuildLease, source_revision_snapshot +from polylogue.storage.introspection import table_exists +from polylogue.storage.sqlite.archive_tiers import ARCHIVE_VERSION_BY_TIER +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.migration_runner import ( + validate_backup_manifest_covers_derived_tier, + validate_migration_backup_manifest, +) +from polylogue.version import VERSION_INFO + + +class RecoveryOperation(StrEnum): + RESET_CENSUS = "reset_raw_authority_census" + PRUNE_INDEX_SEEDS = "prune_orphaned_index_revision_seeds" + + +PLAN_FORMAT = "polylogue.raw-authority-recovery-plan.v1" +RECEIPT_FORMAT = "polylogue.raw-authority-recovery-receipt.v1" +RECOVERY_DIRNAME = "raw-authority-recovery" +_RESET_TABLES = ( + "raw_authority_blockers", + "raw_authority_census_plans", + "raw_authority_census_post_plans", + "raw_authority_plans", + "raw_authority_censuses", +) +_INDEX_TARGETS = ("raw_revision_heads", "raw_revision_applications") + + +class RawAuthorityRecoveryError(RuntimeError): + """A recovery plan or apply could not prove its safety contract.""" + + +def _canonical_bytes(payload: object) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _digest(payload: object) -> str: + return hashlib.sha256(_canonical_bytes(payload)).hexdigest() + + +def _file_fingerprint(path: Path) -> dict[str, object]: + try: + stat = path.stat() + except OSError as exc: + raise RawAuthorityRecoveryError(f"recovery tier is not readable: {path}") from exc + digest = hashlib.sha256() + try: + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + except OSError as exc: + raise RawAuthorityRecoveryError(f"could not fingerprint recovery tier: {path}") from exc + return { + "path": str(path.resolve(strict=False)), + "size_bytes": stat.st_size, + "sha256": digest.hexdigest(), + "device": stat.st_dev, + "inode": stat.st_ino, + } + + +def _pointer_fingerprint(root: Path) -> dict[str, object]: + pointer = root / ".index-active-pointer" + if not pointer.exists() and not pointer.is_symlink(): + return {"path": str(pointer), "exists": False, "text": None} + try: + text = pointer.read_text(encoding="utf-8") + except OSError as exc: + raise RawAuthorityRecoveryError(f"active index pointer is unreadable: {pointer}") from exc + return {"path": str(pointer), "exists": True, "text": text} + + +def _quote_identifier(name: str) -> str: + if not name or any(char not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" for char in name): + raise RawAuthorityRecoveryError(f"unexpected SQLite identifier: {name!r}") + return f'"{name}"' + + +def _ensure_tables(conn: sqlite3.Connection, names: tuple[str, ...], *, tier: str) -> None: + missing = [name for name in names if not table_exists(conn, name)] + if missing: + raise RawAuthorityRecoveryError(f"{tier} database is missing required table(s): {', '.join(missing)}") + + +def _value_for_digest(value: object) -> object: + if isinstance(value, bytes): + return {"type": "bytes", "hex": value.hex()} + if value is None or isinstance(value, (str, int, float, bool)): + return value + return {"type": type(value).__name__, "value": str(value)} + + +def _table_digest(conn: sqlite3.Connection, name: str) -> str: + quoted = _quote_identifier(name) + columns = [str(row[1]) for row in conn.execute(f"PRAGMA table_info({quoted})")] + if not columns: + raise RawAuthorityRecoveryError(f"cannot fingerprint missing or malformed table: {name}") + rows = [[_value_for_digest(value) for value in row] for row in conn.execute(f"SELECT * FROM {quoted}")] + rows.sort(key=_canonical_bytes) + return _digest({"name": name, "columns": columns, "rows": rows}) + + +def _protected_digest(conn: sqlite3.Connection, *, excluded: tuple[str, ...]) -> str: + tables = sorted( + str(row[0]) + for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") + if str(row[0]) not in excluded + ) + return _digest({name: _table_digest(conn, name) for name in tables}) + + +def _schema_versions(root: Path, location: ArchiveLocation) -> dict[str, int]: + paths = { + ArchiveTier.SOURCE.value: root / "source.db", + ArchiveTier.INDEX.value: location.active_index_path, + ArchiveTier.EMBEDDINGS.value: root / "embeddings.db", + ArchiveTier.USER.value: root / "user.db", + ArchiveTier.OPS.value: root / "ops.db", + ArchiveTier.AUDIT.value: root / "audit.db", + } + versions: dict[str, int] = {} + for tier, path in paths.items(): + if not path.is_file(): + raise RawAuthorityRecoveryError(f"archive tier is missing: {path}") + with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn: + versions[tier] = int(conn.execute("PRAGMA user_version").fetchone()[0] or 0) + expected = {tier.value: version for tier, version in ARCHIVE_VERSION_BY_TIER.items()} + if versions != expected: + raise RawAuthorityRecoveryError( + f"archive schema versions are not current: observed={versions}, expected={expected}" + ) + return versions + + +def _archive_identity(root: Path, location: ArchiveLocation) -> dict[str, object]: + from polylogue.storage.archive_identity import ArchiveIdentity + + identity = ArchiveIdentity.resolve_location(location) + return identity.as_dict(unit="raw-authority-recovery") + + +def _generation_identity(root: Path, location: ArchiveLocation) -> dict[str, object]: + payload: dict[str, object] = { + "active_generation": location.active_generation, + "active_index_path": str(location.active_index_path.resolve(strict=False)), + "active_index_stable_id": location.active_index.stable_id, + "pointer": _pointer_fingerprint(root), + } + metadata = location.active_index_path.parent / "generation.json" + if metadata.is_file(): + try: + decoded = json.loads(metadata.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise RawAuthorityRecoveryError(f"active generation metadata is malformed: {metadata}") from exc + if not isinstance(decoded, dict): + raise RawAuthorityRecoveryError(f"active generation metadata is not an object: {metadata}") + payload["metadata"] = {str(key): value for key, value in decoded.items()} + return payload + + +def _validate_ledger(conn: sqlite3.Connection) -> None: + _ensure_tables(conn, ("raw_authority_parser_census", *_RESET_TABLES, "raw_sessions"), tier="source") + for table, columns in { + "raw_authority_censuses": ("scope_json", "residual_json"), + "raw_authority_plans": ( + "input_raw_ids_json", + "logical_keys_json", + "authority_witness_json", + "source_preconditions_json", + "index_preconditions_json", + ), + "raw_authority_census_plans": ("application_receipt_json",), + "raw_authority_blockers": ("expected_json", "observed_json"), + }.items(): + quoted = _quote_identifier(table) + for row in conn.execute(f"SELECT * FROM {quoted}"): + names = [str(item[1]) for item in conn.execute(f"PRAGMA table_info({quoted})")] + values = dict(zip(names, row, strict=True)) + for column in columns: + try: + decoded = json.loads(str(values[column])) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise RawAuthorityRecoveryError(f"raw-authority ledger has malformed {table}.{column}") from exc + if not isinstance(decoded, (dict, list)): + raise RawAuthorityRecoveryError(f"raw-authority ledger has unexpected {table}.{column} JSON class") + if list(conn.execute("PRAGMA foreign_key_check")): + raise RawAuthorityRecoveryError("raw-authority ledger has foreign-key violations") + + +def _validate_integrity(conn: sqlite3.Connection, *, tier: str) -> None: + quick_check = tuple(str(row[0]) for row in conn.execute("PRAGMA quick_check")) + foreign_keys = tuple(tuple(str(value) for value in row) for row in conn.execute("PRAGMA foreign_key_check")) + if quick_check != ("ok",): + raise RawAuthorityRecoveryError(f"{tier} database quick_check failed: {quick_check}") + if foreign_keys: + raise RawAuthorityRecoveryError(f"{tier} database has foreign-key violations: {foreign_keys[:3]}") + + +def _count_tables(conn: sqlite3.Connection, names: tuple[str, ...]) -> dict[str, int]: + _ensure_tables(conn, names, tier="archive") + return {name: int(conn.execute(f"SELECT COUNT(*) FROM {_quote_identifier(name)}").fetchone()[0]) for name in names} + + +def _index_candidates(conn: sqlite3.Connection) -> dict[str, tuple[str, ...]]: + _ensure_tables(conn, _INDEX_TARGETS, tier="index") + heads = tuple( + str(row[0]) + for row in conn.execute( + "SELECT h.logical_source_key FROM raw_revision_heads AS h " + "WHERE NOT EXISTS (SELECT 1 FROM src.raw_sessions AS r WHERE r.raw_id = h.accepted_raw_id) " + "ORDER BY h.logical_source_key" + ) + ) + applications = tuple( + str(row[0]) + for row in conn.execute( + "SELECT a.decision_id FROM raw_revision_applications AS a " + "WHERE NOT EXISTS (SELECT 1 FROM src.raw_sessions AS r WHERE r.raw_id = a.raw_id) " + "ORDER BY a.decision_id" + ) + ) + if any(not value for value in (*heads, *applications)): + raise RawAuthorityRecoveryError("orphaned index revision seed has an empty primary key") + return {"raw_revision_heads": heads, "raw_revision_applications": applications} + + +def _validate_backup(path: Path | None, *, tier: ArchiveTier, connection: sqlite3.Connection) -> dict[str, object]: + if path is None: + raise RawAuthorityRecoveryError(f"{tier.value}-tier backup authority is required for apply") + manifest = path.expanduser().resolve(strict=False) + try: + receipt = ( + validate_migration_backup_manifest(manifest, tier, connection=connection) + if tier in {ArchiveTier.SOURCE, ArchiveTier.USER, ArchiveTier.AUDIT} + else validate_backup_manifest_covers_derived_tier(manifest, tier, connection=connection) + ) + except Exception as exc: + raise RawAuthorityRecoveryError(f"backup authority refused for {tier.value}: {exc}") from exc + receipt = receipt.resolve(strict=False) + return { + "tier": tier.value, + "manifest_path": str(manifest), + "manifest_sha256": _file_fingerprint(manifest)["sha256"], + "receipt_path": str(receipt), + "receipt_sha256": _file_fingerprint(receipt)["sha256"], + } + + +def _backup_from_plan(plan: RawAuthorityRecoveryPlan, *, connection: sqlite3.Connection) -> None: + authority = plan.backup_authority + if not isinstance(authority, dict): + raise RawAuthorityRecoveryError("apply plan has no verified backup authority") + manifest_path = Path(str(authority.get("manifest_path", ""))) + if not manifest_path.is_file() or _file_fingerprint(manifest_path)["sha256"] != authority.get("manifest_sha256"): + raise RawAuthorityRecoveryError("authorized backup manifest changed or is missing") + receipt_path = Path(str(authority.get("receipt_path", ""))) + if not receipt_path.is_file() or _file_fingerprint(receipt_path)["sha256"] != authority.get("receipt_sha256"): + raise RawAuthorityRecoveryError("authorized backup verification receipt changed or is missing") + tier = ArchiveTier(str(authority.get("tier", ""))) + refreshed = _validate_backup(manifest_path, tier=tier, connection=connection) + if refreshed != authority: + raise RawAuthorityRecoveryError("backup authority no longer matches the recovery plan") + + +def _postflight(conn: sqlite3.Connection, *, protected_digest: str, excluded: tuple[str, ...]) -> dict[str, object]: + quick_check = tuple(str(row[0]) for row in conn.execute("PRAGMA quick_check")) + foreign_keys = tuple(tuple(str(value) for value in row) for row in conn.execute("PRAGMA foreign_key_check")) + actual_protected = _protected_digest(conn, excluded=excluded) + if quick_check != ("ok",): + raise RawAuthorityRecoveryError(f"postflight quick_check failed: {quick_check}") + if foreign_keys: + raise RawAuthorityRecoveryError(f"postflight foreign_key_check failed: {foreign_keys[:3]}") + if actual_protected != protected_digest: + raise RawAuthorityRecoveryError("postflight changed an unrelated table") + return { + "quick_check": list(quick_check), + "foreign_key_check": [list(row) for row in foreign_keys], + "protected_digest": actual_protected, + } + + +@dataclass(frozen=True, slots=True) +class RawAuthorityRecoveryPlan: + operation_id: str + operation: str + archive_root: str + archive_identity: dict[str, object] + archive_identity_digest: str + schema_versions: dict[str, int] + code_sha: str + source_fingerprint: dict[str, object] + index_fingerprint: dict[str, object] + source_snapshot: str + active_generation: dict[str, object] + before_counts: dict[str, int] + candidate_keys: dict[str, tuple[str, ...]] + protected_digest: str + backup_authority: dict[str, object] | None + receipt_path: str + plan_digest: str + + def _payload(self) -> dict[str, object]: + return { + "format": PLAN_FORMAT, + "operation_id": self.operation_id, + "operation": self.operation, + "archive_root": self.archive_root, + "archive_identity": self.archive_identity, + "archive_identity_digest": self.archive_identity_digest, + "schema_versions": self.schema_versions, + "code_sha": self.code_sha, + "source_fingerprint": self.source_fingerprint, + "index_fingerprint": self.index_fingerprint, + "source_snapshot": self.source_snapshot, + "active_generation": self.active_generation, + "before_counts": self.before_counts, + "candidate_keys": {key: list(value) for key, value in self.candidate_keys.items()}, + "protected_digest": self.protected_digest, + "backup_authority": self.backup_authority, + "receipt_path": self.receipt_path, + } + + def to_dict(self) -> dict[str, object]: + return {**self._payload(), "plan_digest": self.plan_digest} + + @classmethod + def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: + if payload.get("format") != PLAN_FORMAT: + raise RawAuthorityRecoveryError("unsupported or missing raw-authority recovery plan format") + expected = payload.get("plan_digest") + actual = _digest({key: value for key, value in payload.items() if key != "plan_digest"}) + if not isinstance(expected, str) or expected != actual: + raise RawAuthorityRecoveryError("raw-authority recovery plan digest is invalid") + candidates_raw = payload.get("candidate_keys") + if not isinstance(candidates_raw, dict): + raise RawAuthorityRecoveryError("raw-authority recovery plan candidate keys are malformed") + if any(not isinstance(value_list, list) for value_list in candidates_raw.values()): + raise RawAuthorityRecoveryError("raw-authority recovery plan candidate keys are malformed") + candidates = { + str(key): tuple(str(value) for value in cast(list[object], value_list)) + for key, value_list in candidates_raw.items() + } + required = ( + "operation_id", + "operation", + "archive_root", + "archive_identity", + "archive_identity_digest", + "schema_versions", + "code_sha", + "source_fingerprint", + "index_fingerprint", + "source_snapshot", + "active_generation", + "before_counts", + "protected_digest", + "receipt_path", + ) + if any(key not in payload for key in required): + raise RawAuthorityRecoveryError("raw-authority recovery plan is missing a required field") + return cls( + operation_id=str(payload["operation_id"]), + operation=str(payload["operation"]), + archive_root=str(payload["archive_root"]), + archive_identity=cast(dict[str, object], payload["archive_identity"]), + archive_identity_digest=str(payload["archive_identity_digest"]), + schema_versions=_int_mapping(payload["schema_versions"], field="schema_versions"), + code_sha=str(payload["code_sha"]), + source_fingerprint=cast(dict[str, object], payload["source_fingerprint"]), + index_fingerprint=cast(dict[str, object], payload["index_fingerprint"]), + source_snapshot=str(payload["source_snapshot"]), + active_generation=cast(dict[str, object], payload["active_generation"]), + before_counts=_int_mapping(payload["before_counts"], field="before_counts"), + candidate_keys=candidates, + protected_digest=str(payload["protected_digest"]), + backup_authority=( + cast(dict[str, object], payload["backup_authority"]) + if isinstance(payload.get("backup_authority"), dict) + else None + ), + receipt_path=str(payload["receipt_path"]), + plan_digest=actual, + ) + + +@dataclass(frozen=True, slots=True) +class RawAuthorityRecoveryReport: + plan: RawAuthorityRecoveryPlan + applied: bool + status: str + receipt_path: Path | None = None + after_counts: dict[str, int] | None = None + postflight: dict[str, object] | None = None + + def to_dict(self) -> dict[str, object]: + return { + "operation_id": self.plan.operation_id, + "operation": self.plan.operation, + "plan": self.plan.to_dict(), + "applied": self.applied, + "status": self.status, + "receipt_path": str(self.receipt_path) if self.receipt_path is not None else None, + "after_counts": self.after_counts, + "postflight": self.postflight, + } + + +def _default_receipt_path(root: Path, operation_id: str) -> Path: + return root / ".maintenance-state" / RECOVERY_DIRNAME / f"{operation_id}.receipt.json" + + +def _read_json(path: Path) -> dict[str, object]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise RawAuthorityRecoveryError(f"recovery plan or receipt is unreadable: {path}") from exc + if not isinstance(payload, dict): + raise RawAuthorityRecoveryError(f"recovery artifact is not a JSON object: {path}") + return {str(key): value for key, value in payload.items()} + + +def _int_mapping(payload: object, *, field: str) -> dict[str, int]: + if not isinstance(payload, dict) or any(not isinstance(value, int) for value in payload.values()): + raise RawAuthorityRecoveryError(f"raw-authority recovery plan field {field!r} is malformed") + return {str(key): int(value) for key, value in payload.items()} + + +def _write_immutable(path: Path, payload: dict[str, object], *, digest_field: str) -> Path: + body = {key: value for key, value in payload.items() if key != digest_field} + expected = _digest(body) + stamped = {**body, digest_field: expected} + path = path.expanduser().resolve(strict=False) + if path.exists(): + existing = _read_json(path) + if existing != stamped: + raise RawAuthorityRecoveryError( + f"immutable recovery artifact already exists with different content: {path}" + ) + return path + path.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)[1]) + try: + temporary.write_text(json.dumps(stamped, indent=2, sort_keys=True) + "\n", encoding="utf-8") + with temporary.open("rb") as handle: + os.fsync(handle.fileno()) + os.link(temporary, path) + except FileExistsError: + existing = _read_json(path) + if existing != stamped: + raise RawAuthorityRecoveryError(f"immutable recovery artifact race changed content: {path}") from None + finally: + temporary.unlink(missing_ok=True) + return path + + +def write_recovery_plan(plan: RawAuthorityRecoveryPlan, path: Path) -> Path: + return _write_immutable(path, plan.to_dict(), digest_field="plan_digest") + + +def _load_plan(path: Path) -> RawAuthorityRecoveryPlan: + return RawAuthorityRecoveryPlan.from_dict(_read_json(path)) + + +def _build_plan( + archive_root: Path, + *, + operation: RecoveryOperation, + operation_id: str, + backup_manifest: Path | None, + receipt_path: Path | None, +) -> RawAuthorityRecoveryPlan: + root = archive_root.expanduser().resolve(strict=False) + location = ArchiveLocation.resolve(root) + source_db = root / "source.db" + index_db = location.active_index_path + if not source_db.is_file() or not index_db.is_file(): + raise FileNotFoundError(source_db if not source_db.is_file() else index_db) + schema_versions = _schema_versions(root, location) + code_sha = VERSION_INFO.commit + if not code_sha: + raise RawAuthorityRecoveryError("recovery requires an exact build code SHA") + identity = _archive_identity(root, location) + identity_digest = _digest(identity) + with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as source: + source.row_factory = sqlite3.Row + source_counts = _count_tables(source, _RESET_TABLES) + if operation is RecoveryOperation.RESET_CENSUS: + _validate_ledger(source) + _validate_integrity(source, tier="source") + counts = source_counts + candidate_keys: dict[str, tuple[str, ...]] = {} + protected = _protected_digest(source, excluded=_RESET_TABLES) + else: + with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as index: + index.row_factory = sqlite3.Row + index.execute("ATTACH DATABASE ? AS src", (str(source_db),)) + _validate_integrity(index, tier="active index") + candidate_keys = _index_candidates(index) + counts = _count_tables(index, _INDEX_TARGETS) + protected = _protected_digest(index, excluded=_INDEX_TARGETS) + source_snapshot = source_revision_snapshot(root) + backup_authority: dict[str, object] | None = None + if backup_manifest is not None: + target = ArchiveTier.SOURCE if operation is RecoveryOperation.RESET_CENSUS else ArchiveTier.INDEX + with closing( + sqlite3.connect(f"file:{source_db if target is ArchiveTier.SOURCE else index_db}?mode=ro", uri=True) + ) as conn: + backup_authority = _validate_backup(backup_manifest, tier=target, connection=conn) + receipt = receipt_path or _default_receipt_path(root, operation_id) + payload: dict[str, object] = { + "format": PLAN_FORMAT, + "operation_id": operation_id, + "operation": operation.value, + "archive_root": str(root), + "archive_identity": identity, + "archive_identity_digest": identity_digest, + "schema_versions": schema_versions, + "code_sha": code_sha, + "source_fingerprint": _file_fingerprint(source_db), + "index_fingerprint": _file_fingerprint(index_db), + "source_snapshot": source_snapshot, + "active_generation": _generation_identity(root, location), + "before_counts": counts, + "candidate_keys": {key: list(value) for key, value in candidate_keys.items()}, + "protected_digest": protected, + "backup_authority": backup_authority, + "receipt_path": str(receipt.resolve(strict=False)), + } + return RawAuthorityRecoveryPlan( + operation_id=operation_id, + operation=operation.value, + archive_root=str(root), + archive_identity=identity, + archive_identity_digest=identity_digest, + schema_versions=schema_versions, + code_sha=code_sha, + source_fingerprint=cast(dict[str, object], payload["source_fingerprint"]), + index_fingerprint=cast(dict[str, object], payload["index_fingerprint"]), + source_snapshot=source_snapshot, + active_generation=cast(dict[str, object], payload["active_generation"]), + before_counts=counts, + candidate_keys=candidate_keys, + protected_digest=protected, + backup_authority=backup_authority, + receipt_path=str(receipt.resolve(strict=False)), + plan_digest=_digest(payload), + ) + + +def inspect_raw_authority_recovery( + archive_root: Path, + operation: RecoveryOperation | str, + *, + operation_id: str | None = None, + backup_manifest: Path | None = None, + receipt_path: Path | None = None, +) -> RawAuthorityRecoveryPlan: + """Build a read-only, exact plan for one recovery operation.""" + selected = RecoveryOperation(operation) + return _build_plan( + archive_root, + operation=selected, + operation_id=operation_id or f"raw-authority-recovery:{uuid.uuid4().hex}", + backup_manifest=backup_manifest, + receipt_path=receipt_path, + ) + + +def _offline_config(root: Path) -> Config: + return Config(archive_root=root, render_root=render_root(), sources=[]) + + +def _require_apply_preconditions(root: Path) -> None: + if running_daemon_pid(_offline_config(root)) is not None: + raise RawAuthorityRecoveryError("refusing raw-authority recovery while polylogued is running") + if reason := offline_maintenance_block_reason(_offline_config(root), active=True, dry_run=False): + raise RawAuthorityRecoveryError(reason) + + +def _same(value: object, expected: object, field: str) -> None: + if value != expected: + raise RawAuthorityRecoveryError(f"recovery plan is stale: {field} changed") + + +def _revalidate_common(plan: RawAuthorityRecoveryPlan, root: Path, location: ArchiveLocation) -> None: + if str(root.resolve(strict=False)) != plan.archive_root: + raise RawAuthorityRecoveryError("recovery plan names a different archive root") + _same(VERSION_INFO.commit, plan.code_sha, "code SHA") + _same(_schema_versions(root, location), plan.schema_versions, "schema versions") + _same(_archive_identity(root, location), plan.archive_identity, "archive identity") + _same(_digest(plan.archive_identity), plan.archive_identity_digest, "archive identity digest") + _same(_generation_identity(root, location), plan.active_generation, "active index generation/pointer") + _same(_file_fingerprint(root / "source.db"), plan.source_fingerprint, "source database") + _same(_file_fingerprint(location.active_index_path), plan.index_fingerprint, "active index database") + _same(source_revision_snapshot(root), plan.source_snapshot, "source snapshot") + + +def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: + root = Path(plan.archive_root) + _require_apply_preconditions(root) + location = ArchiveLocation.resolve(root) + operation = RecoveryOperation(plan.operation) + source_db = root / "source.db" + index_db = location.active_index_path + excluded = _RESET_TABLES if operation is RecoveryOperation.RESET_CENSUS else _INDEX_TARGETS + before_counts = dict(plan.before_counts) + if operation is RecoveryOperation.RESET_CENSUS: + with closing(sqlite3.connect(source_db)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("BEGIN IMMEDIATE") + try: + _revalidate_common(plan, root, location) + _validate_ledger(conn) + _validate_integrity(conn, tier="source") + _same(_count_tables(conn, _RESET_TABLES), before_counts, "census ledger counts") + _same(_protected_digest(conn, excluded=excluded), plan.protected_digest, "protected source rows") + _backup_from_plan(plan, connection=conn) + for table in _RESET_TABLES: + conn.execute(f"DELETE FROM {_quote_identifier(table)}") + after_counts = _count_tables(conn, _RESET_TABLES) + if any(after_counts.values()): + raise RawAuthorityRecoveryError("census reset postflight left ledger rows behind") + postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) + conn.commit() + except Exception: + conn.rollback() + raise + else: + with closing(sqlite3.connect(index_db)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("ATTACH DATABASE ? AS src", (str(source_db),)) + conn.execute("BEGIN IMMEDIATE") + try: + _revalidate_common(plan, root, location) + _validate_integrity(conn, tier="active index") + _same(_index_candidates(conn), plan.candidate_keys, "orphaned index candidate rows") + _same(_protected_digest(conn, excluded=excluded), plan.protected_digest, "protected index rows") + _backup_from_plan(plan, connection=conn) + for key in plan.candidate_keys["raw_revision_heads"]: + if ( + conn.execute("DELETE FROM raw_revision_heads WHERE logical_source_key = ?", (key,)).rowcount + != 1 + ): + raise RawAuthorityRecoveryError(f"expected exactly one raw_revision_heads row for {key!r}") + for key in plan.candidate_keys["raw_revision_applications"]: + if ( + conn.execute("DELETE FROM raw_revision_applications WHERE decision_id = ?", (key,)).rowcount + != 1 + ): + raise RawAuthorityRecoveryError( + f"expected exactly one raw_revision_applications row for {key!r}" + ) + after_counts = { + key: int(conn.execute(f"SELECT COUNT(*) FROM {_quote_identifier(key)}").fetchone()[0]) + for key in _INDEX_TARGETS + } + expected_after = {key: before_counts[key] - len(plan.candidate_keys[key]) for key in _INDEX_TARGETS} + if after_counts != expected_after: + raise RawAuthorityRecoveryError("index-seed prune postflight changed an unexpected row class") + postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) + conn.commit() + except Exception: + conn.rollback() + raise + receipt_payload: dict[str, object] = { + "format": RECEIPT_FORMAT, + "operation_id": plan.operation_id, + "operation": plan.operation, + "archive_root": plan.archive_root, + "plan_digest": plan.plan_digest, + "code_sha": plan.code_sha, + "archive_identity": plan.archive_identity, + "schema_versions": plan.schema_versions, + "active_generation": plan.active_generation, + "source_snapshot_before": plan.source_snapshot, + "source_snapshot_after": source_revision_snapshot(root), + "source_fingerprint_before": plan.source_fingerprint, + "index_fingerprint_before": plan.index_fingerprint, + "source_fingerprint_after": _file_fingerprint(root / "source.db"), + "index_fingerprint_after": _file_fingerprint(ArchiveLocation.resolve(root).active_index_path), + "before_counts": before_counts, + "after_counts": after_counts, + "candidate_keys": {key: list(value) for key, value in plan.candidate_keys.items()}, + "backup_authority": plan.backup_authority, + "protected_digest_before": plan.protected_digest, + "protected_digest_after": postflight["protected_digest"], + "postflight": postflight, + } + receipt_path = _write_immutable(Path(plan.receipt_path), receipt_payload, digest_field="receipt_sha256") + return RawAuthorityRecoveryReport( + plan=plan, + applied=True, + status="applied", + receipt_path=receipt_path, + after_counts=after_counts, + postflight=postflight, + ) + + +@dataclass(frozen=True, slots=True) +class _RecoveryArgs: + archive_root: Path + operation: RecoveryOperation + operation_id: str + expected_plan_digest: str + backup_manifest: Path | None + receipt_path: Path + + +@dataclass(frozen=True, slots=True) +class _RecoveryActuator: + operation: str + recovery_operation: RecoveryOperation + destructive_class: DestructiveClass = "reset" + required_confirmation: ConfirmationStrength = "confirm_flag" + + def prepare(self, args: _RecoveryArgs) -> MutationPlan: + live = inspect_raw_authority_recovery( + args.archive_root, + args.operation, + operation_id=args.operation_id, + backup_manifest=args.backup_manifest, + receipt_path=args.receipt_path, + ) + return build_plan( + operation=self.operation, + destructive_class=self.destructive_class, + target_refs=( + make_target_ref( + "source", + args.operation.value, + ), + ), + affected_tiers=("source",) if args.operation is RecoveryOperation.RESET_CENSUS else ("index",), + reversible=False, + context={"recovery_plan_digest": live.plan_digest, "operation_id": args.operation_id}, + ) + + def apply(self, plan: MutationPlan, args: _RecoveryArgs) -> MutationReceipt: + del plan + live = inspect_raw_authority_recovery( + args.archive_root, + args.operation, + operation_id=args.operation_id, + backup_manifest=args.backup_manifest, + receipt_path=args.receipt_path, + ) + if live.plan_digest != args.expected_plan_digest: + raise PlanStaleError("raw-authority recovery plan digest changed before apply") + report = _apply_plan(live) + return MutationReceipt( + operation=self.operation, + plan_hash=args.expected_plan_digest, + status="applied", + target_refs=( + make_target_ref( + "source", + args.operation.value, + ), + ), + affected_count=sum(report.plan.before_counts.values()), + detail=None, + receipt_ref=str(report.receipt_path) if report.receipt_path is not None else None, + applied_at="recovery", + domain_receipt=report.to_dict(), + operation_id=args.operation_id, + ) + + +class ResetRawAuthorityCensusActuator(_RecoveryActuator): + def __init__(self) -> None: + super().__init__("mutate-reset-raw-authority-census", RecoveryOperation.RESET_CENSUS) + + +class PruneOrphanedIndexRevisionSeedsActuator(_RecoveryActuator): + def __init__(self) -> None: + super().__init__("mutate-prune-orphaned-index-revision-seeds", RecoveryOperation.PRUNE_INDEX_SEEDS) + + +def _receipt_for_plan(plan: RawAuthorityRecoveryPlan) -> dict[str, object] | None: + path = Path(plan.receipt_path) + if not path.is_file(): + return None + payload = _read_json(path) + expected = payload.get("receipt_sha256") + if not isinstance(expected, str) or expected != _digest( + {key: value for key, value in payload.items() if key != "receipt_sha256"} + ): + raise RawAuthorityRecoveryError("existing recovery receipt has an invalid self-hash") + if payload.get("plan_digest") != plan.plan_digest or payload.get("operation_id") != plan.operation_id: + raise RawAuthorityRecoveryError("existing recovery receipt belongs to another operation or plan") + return payload + + +def _validate_existing_receipt(plan: RawAuthorityRecoveryPlan, receipt: dict[str, object]) -> None: + root = Path(plan.archive_root) + location = ArchiveLocation.resolve(root) + _same(_schema_versions(root, location), plan.schema_versions, "schema versions") + _same(_archive_identity(root, location), plan.archive_identity, "archive identity") + _same(_generation_identity(root, location), receipt.get("active_generation"), "active index generation/pointer") + _same(source_revision_snapshot(root), receipt.get("source_snapshot_after"), "source snapshot") + _same(_file_fingerprint(root / "source.db"), receipt.get("source_fingerprint_after"), "source database") + _same( + _file_fingerprint(location.active_index_path), receipt.get("index_fingerprint_after"), "active index database" + ) + postflight = receipt.get("postflight") + if not isinstance(postflight, dict) or receipt.get("protected_digest_after") != postflight.get("protected_digest"): + raise RawAuthorityRecoveryError("existing recovery receipt has inconsistent postflight evidence") + + +def apply_raw_authority_recovery( + plan: RawAuthorityRecoveryPlan | Path, + *, + backup_manifest: Path | None = None, +) -> RawAuthorityRecoveryReport: + """Apply one exact plan through the named actuator lifecycle.""" + selected = _load_plan(plan) if isinstance(plan, Path) else plan + if backup_manifest is not None and ( + selected.backup_authority is None + or str(backup_manifest.resolve(strict=False)) != selected.backup_authority.get("manifest_path") + ): + raise RawAuthorityRecoveryError("apply backup manifest does not match the plan authority") + existing = _receipt_for_plan(selected) + if existing is not None: + _require_apply_preconditions(Path(selected.archive_root)) + _validate_existing_receipt(selected, existing) + return RawAuthorityRecoveryReport( + plan=selected, + applied=False, + status="already_satisfied", + receipt_path=Path(selected.receipt_path), + after_counts=cast(dict[str, int], existing.get("after_counts")), + postflight=cast(dict[str, object], existing.get("postflight")), + ) + if selected.backup_authority is None: + raise RawAuthorityRecoveryError("apply requires a dry-run plan with verified backup authority") + operation = RecoveryOperation(selected.operation) + root = Path(selected.archive_root) + args = _RecoveryArgs( + archive_root=root, + operation=operation, + operation_id=selected.operation_id, + expected_plan_digest=selected.plan_digest, + backup_manifest=Path(str(selected.backup_authority["manifest_path"])), + receipt_path=Path(selected.receipt_path), + ) + actuator: _RecoveryActuator = ( + ResetRawAuthorityCensusActuator() + if operation is RecoveryOperation.RESET_CENSUS + else PruneOrphanedIndexRevisionSeedsActuator() + ) + executor = OperationExecutor() + try: + location = ArchiveLocation.resolve(root) + prepared = executor.prepare(actuator, args) + if prepared.context.get("recovery_plan_digest") != selected.plan_digest: + raise PlanStaleError("recovery plan is stale before lease acquisition") + authorization = executor.authorize( + actuator, + prepared, + actor="cli:maintenance", + role="maintenance", + capability="archive.raw_authority_recovery", + confirmation_strength="confirm_flag", + ) + with OwnedArchiveLocation.acquire( + location, owner_id=f"raw-authority-recovery:{selected.operation_id}" + ) as owned: + current_location = ArchiveLocation.resolve(root) + assert_owns_archive_location(owned, current_location) + with RebuildLease(root): + result = executor.execute(actuator, prepared, authorization, args) + except ( + ArchiveLocationError, + ArchiveOwnershipError, + FileNotFoundError, + OSError, + sqlite3.Error, + ValueError, + MutationTransactionError, + ) as exc: + if isinstance(exc, RawAuthorityRecoveryError): + raise + raise RawAuthorityRecoveryError(str(exc)) from exc + domain = dict(result.domain_receipt) + return RawAuthorityRecoveryReport( + plan=selected, + applied=result.status == "applied", + status=result.status, + receipt_path=Path(str(domain["receipt_path"])) if domain.get("receipt_path") else None, + after_counts=cast(dict[str, int] | None, domain.get("after_counts")), + postflight=cast(dict[str, object] | None, domain.get("postflight")), + ) + + +__all__ = [ + "PLAN_FORMAT", + "RECEIPT_FORMAT", + "PruneOrphanedIndexRevisionSeedsActuator", + "RawAuthorityRecoveryError", + "RawAuthorityRecoveryPlan", + "RawAuthorityRecoveryReport", + "RecoveryOperation", + "ResetRawAuthorityCensusActuator", + "apply_raw_authority_recovery", + "inspect_raw_authority_recovery", + "write_recovery_plan", +] diff --git a/polylogue/maintenance/raw_authority_reset.py b/polylogue/maintenance/raw_authority_reset.py index 90503897f1..033cade504 100644 --- a/polylogue/maintenance/raw_authority_reset.py +++ b/polylogue/maintenance/raw_authority_reset.py @@ -29,11 +29,11 @@ from dataclasses import dataclass from pathlib import Path -from polylogue.config import Config -from polylogue.maintenance.offline_guard import offline_maintenance_block_reason -from polylogue.paths import render_root -from polylogue.storage.raw_authority import prune_orphaned_index_revision_seeds as _prune_orphaned_index_revision_seeds -from polylogue.storage.raw_authority import reset_raw_authority_census_ledger +from polylogue.maintenance.raw_authority_recovery import ( + RecoveryOperation, + apply_raw_authority_recovery, + inspect_raw_authority_recovery, +) @dataclass(frozen=True, slots=True) @@ -48,34 +48,28 @@ class RawAuthorityResetReport: applied: bool -def _offline_config(archive_root: Path) -> Config: - return Config(archive_root=archive_root, render_root=render_root(), sources=[]) - - def reset_raw_authority_census( archive_root: Path, *, backup_manifest: Path | None = None, dry_run: bool = True, ) -> RawAuthorityResetReport: - """Empty the census planning ledger. ``dry_run`` reports counts only.""" - if not dry_run and ( - reason := offline_maintenance_block_reason(_offline_config(archive_root), active=True, dry_run=False) - ): - raise RuntimeError(reason) - before = reset_raw_authority_census_ledger( + """Inspect or apply the guarded census-ledger recovery route.""" + plan = inspect_raw_authority_recovery( archive_root, + RecoveryOperation.RESET_CENSUS, backup_manifest=backup_manifest, - dry_run=dry_run, ) + report = apply_raw_authority_recovery(plan, backup_manifest=backup_manifest) if not dry_run else None + counts = plan.before_counts return RawAuthorityResetReport( - censuses=before.censuses, - plans=before.plans, - blockers=before.blockers, - census_plans=before.census_plans, - census_post_plans=before.census_post_plans, - applied=not dry_run, + censuses=counts["raw_authority_censuses"], + plans=counts["raw_authority_plans"], + blockers=counts["raw_authority_blockers"], + census_plans=counts["raw_authority_census_plans"], + census_post_plans=counts["raw_authority_census_post_plans"], + applied=report is not None and report.status == "applied", ) @@ -88,7 +82,12 @@ class IndexSeedPruneReport: applied: bool -def prune_orphaned_index_revision_seeds(archive_root: Path, *, dry_run: bool = True) -> IndexSeedPruneReport: +def prune_orphaned_index_revision_seeds( + archive_root: Path, + *, + backup_manifest: Path | None = None, + dry_run: bool = True, +) -> IndexSeedPruneReport: """Delete index raw-frontier seeds whose raw is gone from the source tier. ``raw_revision_heads`` / ``raw_revision_applications`` are the index's @@ -99,13 +98,14 @@ def prune_orphaned_index_revision_seeds(archive_root: Path, *, dry_run: bool = T ``raw_id`` no longer exists in ``source.raw_sessions`` restores a clean frontier; seeds for present raws are untouched. """ - if not dry_run and ( - reason := offline_maintenance_block_reason(_offline_config(archive_root), active=True, dry_run=False) - ): - raise RuntimeError(reason) - counts = _prune_orphaned_index_revision_seeds(archive_root, dry_run=dry_run) + plan = inspect_raw_authority_recovery( + archive_root, + RecoveryOperation.PRUNE_INDEX_SEEDS, + backup_manifest=backup_manifest, + ) + report = apply_raw_authority_recovery(plan, backup_manifest=backup_manifest) if not dry_run else None return IndexSeedPruneReport( - revision_heads=counts.revision_heads, - revision_applications=counts.revision_applications, - applied=not dry_run, + revision_heads=len(plan.candidate_keys["raw_revision_heads"]), + revision_applications=len(plan.candidate_keys["raw_revision_applications"]), + applied=report is not None and report.status == "applied", ) diff --git a/polylogue/operations/specs.py b/polylogue/operations/specs.py index 300a5c32c1..2107e87ce5 100644 --- a/polylogue/operations/specs.py +++ b/polylogue/operations/specs.py @@ -945,6 +945,52 @@ def to_dict(self) -> JSONDocumentList: safety_guards=("write_role_required", "confirmed_before_execute", "explicit_dry_run_evidence"), executor_status="executor-routed", ), + OperationSpec( + name="mutate-reset-raw-authority-census", + kind=OperationKind.MAINTENANCE, + description=( + "Last-resort reset of poisoned raw-authority census bookkeeping. The route is dry-run first, " + "requires an exact source-tier backup-attested plan and stopped-daemon maintenance ownership, " + "and never touches parser census, accepted raws, or blobs." + ), + consumes=("raw_authority_census_ledger",), + produces=("raw_authority_census_recovery_receipt",), + path_targets=("raw-authority-recovery-loop",), + code_refs=( + "polylogue.maintenance.raw_authority_recovery.ResetRawAuthorityCensusActuator", + "polylogue.cli.commands.maintenance._raw_authority_recovery.raw_authority_recovery_command", + ), + surfaces=("cli",), + mutates_state=True, + previewable=True, + idempotent=True, + effects=("DbRead", "DbWrite", "Destructive"), + safety_guards=("write_role_required", "confirmed_before_execute", "explicit_dry_run_evidence"), + executor_status="executor-routed", + ), + OperationSpec( + name="mutate-prune-orphaned-index-revision-seeds", + kind=OperationKind.MAINTENANCE, + description=( + "Remove only active-index raw revision seed rows whose source raws are absent. The route binds " + "the exact active generation, source snapshot, recoverable index backup, and stopped-daemon " + "maintenance ownership; it never performs a broad index reset." + ), + consumes=("raw_revision_heads", "raw_revision_applications", "raw_sessions"), + produces=("raw_authority_index_seed_recovery_receipt",), + path_targets=("raw-authority-recovery-loop",), + code_refs=( + "polylogue.maintenance.raw_authority_recovery.PruneOrphanedIndexRevisionSeedsActuator", + "polylogue.cli.commands.maintenance._raw_authority_recovery.raw_authority_recovery_command", + ), + surfaces=("cli",), + mutates_state=True, + previewable=True, + idempotent=True, + effects=("DbRead", "DbWrite", "Destructive"), + safety_guards=("write_role_required", "confirmed_before_execute", "explicit_dry_run_evidence"), + executor_status="executor-routed", + ), OperationSpec( name="mutate-save-saved-view", kind=OperationKind.MAINTENANCE, diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index 3393e703b6..d8830c85b3 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -24,8 +24,6 @@ from polylogue.core.json import JSONDocument, json_document from polylogue.logging import get_logger from polylogue.storage.archive_identity import ArchiveLocation -from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier -from polylogue.storage.sqlite.migration_runner import validate_migration_backup_manifest #: Fingerprints previously stamped by ``RAW_AUTHORITY_PARSER_FINGERPRINT`` #: whose classification semantics are known to have been superseded by a @@ -75,7 +73,6 @@ RAW_AUTHORITY_CENSUS_HEADER_RETENTION = 256 logger = get_logger(__name__) - def parser_census_logical_keys(logical_keys_json: object) -> tuple[str, ...] | None: """Validate and normalize the durable logical-key receipt payload. @@ -111,7 +108,6 @@ def parser_census_logical_keys(logical_keys_json: object) -> tuple[str, ...] | N "raw_authority_censuses", ) - @dataclass(frozen=True, slots=True) class RawAuthorityCensusResetCounts: """Counts for a reset of derived raw-authority census bookkeeping.""" @@ -2285,7 +2281,14 @@ def reset_raw_authority_census_ledger( backup_manifest: Path | None, dry_run: bool, ) -> RawAuthorityCensusResetCounts: - """Reset derived census bookkeeping after authenticating a source backup.""" + """Return ledger counts for diagnostics. + + Direct storage-layer deletion was intentionally removed. The maintenance + recovery route owns plan binding, offline ownership, backup authority, + postflight, and immutable receipts. Keeping this read-only compatibility + helper prevents callers from mistaking a storage primitive for an operator + authorization boundary. + """ source_db = archive_root / "source.db" if not source_db.is_file(): raise FileNotFoundError(source_db) @@ -2303,12 +2306,9 @@ def reset_raw_authority_census_ledger( ), } if not dry_run: - if backup_manifest is None: - raise ValueError("raw-authority census reset requires a verified source backup manifest") - validate_migration_backup_manifest(backup_manifest, ArchiveTier.SOURCE, connection=conn) - for table in _RESET_LEDGER_TABLES_CHILD_FIRST: - conn.execute(f"DELETE FROM {table}") - conn.commit() + raise RuntimeError( + "direct raw-authority census mutation is disabled; use the guarded maintenance recovery route" + ) return RawAuthorityCensusResetCounts( censuses=counts["raw_authority_censuses"], plans=counts["raw_authority_plans"], @@ -2323,7 +2323,11 @@ def prune_orphaned_index_revision_seeds( *, dry_run: bool, ) -> OrphanedIndexRevisionSeedCounts: - """Prune rebuildable revision seeds that no longer have source authority.""" + """Return orphan counts for diagnostics without mutating the index. + + Apply is deliberately unavailable here. The named maintenance route is + the only authority that may delete these rows. + """ source_db = archive_root / "source.db" index_db = ArchiveLocation.resolve(archive_root).active_index_path if not source_db.is_file() or not index_db.is_file(): @@ -2341,13 +2345,9 @@ def prune_orphaned_index_revision_seeds( ).fetchone()[0] ) if not dry_run: - conn.execute( - "DELETE FROM raw_revision_heads WHERE accepted_raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" - ) - conn.execute( - "DELETE FROM raw_revision_applications WHERE raw_id NOT IN (SELECT raw_id FROM src.raw_sessions)" + raise RuntimeError( + "direct orphaned-index-seed mutation is disabled; use the guarded maintenance recovery route" ) - conn.commit() return OrphanedIndexRevisionSeedCounts( revision_heads=heads, revision_applications=applications, diff --git a/tests/unit/cli/test_maintenance_registration.py b/tests/unit/cli/test_maintenance_registration.py index 5270b27031..a004461a0b 100644 --- a/tests/unit/cli/test_maintenance_registration.py +++ b/tests/unit/cli/test_maintenance_registration.py @@ -9,6 +9,7 @@ from polylogue.cli.commands.maintenance._blob_reference_closure import blob_reference_closure_command from polylogue.cli.commands.maintenance._hook_payload_ref_reconciliation import hook_payload_ref_reconcile_command from polylogue.cli.commands.maintenance._plan import plan_command +from polylogue.cli.commands.maintenance._raw_authority_recovery import raw_authority_recovery_command from polylogue.cli.commands.maintenance._run import run_command from polylogue.cli.commands.maintenance._run_preview import run_preview_command from polylogue.cli.commands.maintenance._status import status_command @@ -38,6 +39,10 @@ def test_maintenance_plan_is_click_command() -> None: assert isinstance(plan_command, click.Command) +def test_raw_authority_recovery_is_click_command() -> None: + assert isinstance(raw_authority_recovery_command, click.Command) + + def test_maintenance_run_is_click_command() -> None: """run is a Click Command on the maintenance group.""" assert isinstance(run_command, click.Command) @@ -64,6 +69,7 @@ def test_maintenance_group_has_plan_and_run() -> None: assert "plan" in cmds assert "run" in cmds assert "run-preview" in cmds + assert "raw-authority-recovery" in cmds def test_maintenance_plan_help_output() -> None: diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 35e6e4529d..916f604dc8 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -1,25 +1,36 @@ -"""Raw-authority census-ledger reset (convergence recovery).""" +"""Real-failure tests for the guarded raw-authority recovery family.""" from __future__ import annotations import hashlib +import json import shutil import sqlite3 from pathlib import Path import pytest +from polylogue.maintenance.raw_authority_recovery import ( + RawAuthorityRecoveryError, + RecoveryOperation, + apply_raw_authority_recovery, + inspect_raw_authority_recovery, +) from polylogue.maintenance.raw_authority_reset import ( prune_orphaned_index_revision_seeds, reset_raw_authority_census, ) from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root -from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier def _seed_ledger(source_db: Path) -> None: with sqlite3.connect(source_db) as conn: - conn.execute("PRAGMA foreign_keys = OFF") # seeding only + conn.execute( + "INSERT INTO raw_authority_parser_census " + "(raw_id, parser_fingerprint, status, logical_keys_json, detail, censused_at_ms) " + "VALUES ('r-keep', 'parser-fp', 'complete', '[\"logical\"]', 'kept', 1)" + ) + conn.execute("PRAGMA foreign_keys = OFF") conn.execute( "INSERT INTO raw_authority_censuses (census_id, sequence_no, scope_json, residual_json, " "parser_fingerprint, mode, lifecycle_status, quiescent, inventory_digest, residual_digest, " @@ -47,38 +58,71 @@ def _seed_ledger(source_db: Path) -> None: ) -def test_reset_empties_ledger_but_preserves_accepted_state(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - initialize_active_archive_root(tmp_path) - source_db = tmp_path / "source.db" - _seed_ledger(source_db) - # Accepted materialization state that MUST survive a census-ledger reset. +def _seed_raw(source_db: Path, raw_id: str) -> None: with sqlite3.connect(source_db) as conn: conn.execute( "INSERT INTO raw_sessions (raw_id, origin, source_path, source_index, blob_hash, blob_size, " - "acquired_at_ms, revision_authority) VALUES ('r-keep','codex-session','/p',0,?,10,1,'byte_proven')", - (b"\x01" * 32,), + "acquired_at_ms, revision_authority) VALUES (?, 'codex-session','/p',0,?,10,1,'byte_proven')", + (raw_id, bytes.fromhex("01" * 32)), ) - dry = reset_raw_authority_census(tmp_path, dry_run=True) - assert dry.applied is False - assert (dry.censuses, dry.plans, dry.blockers, dry.census_plans, dry.census_post_plans) == (1, 1, 1, 1, 1) - with sqlite3.connect(source_db) as conn: - assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone()[0] == 1 # dry: nothing deleted - validated: list[tuple[Path, object]] = [] +def _backup_authority(root: Path, monkeypatch: pytest.MonkeyPatch, *, tier: str) -> Path: + backup = root / f"{tier}-backup" / "manifest.json" + backup.parent.mkdir() + backup.write_text("manifest", encoding="utf-8") + receipt = backup.with_name("verification-receipt.json") + receipt.write_text("receipt", encoding="utf-8") - def validate(manifest: Path, tier: object, *, connection: sqlite3.Connection) -> Path: - validated.append((manifest, tier)) - assert connection.execute("SELECT 1").fetchone() == (1,) - return manifest.with_name("verification-receipt.json") + def validate(_path: Path, _tier: object, *, connection: sqlite3.Connection) -> Path: + assert tuple(connection.execute("SELECT 1").fetchone()) == (1,) + return receipt - monkeypatch.setattr("polylogue.storage.raw_authority.validate_migration_backup_manifest", validate) - manifest = tmp_path / "verified-backup" / "manifest.json" - report = reset_raw_authority_census(tmp_path, backup_manifest=manifest, dry_run=False) - assert report.applied is True - assert validated == [(manifest, ArchiveTier.SOURCE)] + if tier == "source": + monkeypatch.setattr("polylogue.maintenance.raw_authority_recovery.validate_migration_backup_manifest", validate) + else: + monkeypatch.setattr( + "polylogue.maintenance.raw_authority_recovery.validate_backup_manifest_covers_derived_tier", validate + ) + return backup - with sqlite3.connect(source_db) as conn: + +def test_census_reset_reproduces_poisoned_ledger_and_preserves_source_authority( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + + def bypass(*_args: object, **_kwargs: object) -> None: + raise AssertionError("direct storage reset bypass was called") + + monkeypatch.setattr("polylogue.storage.raw_authority.reset_raw_authority_census_ledger", bypass) + + before_parser = ( + sqlite3.connect(tmp_path / "source.db").execute("SELECT * FROM raw_authority_parser_census").fetchall() + ) + dry = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS) + assert dry.before_counts == { + "raw_authority_censuses": 1, + "raw_authority_plans": 1, + "raw_authority_blockers": 1, + "raw_authority_census_plans": 1, + "raw_authority_census_post_plans": 1, + } + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + report = apply_raw_authority_recovery(plan) + assert report.status == "applied" + assert report.postflight == { + "quick_check": ["ok"], + "foreign_key_check": [], + "protected_digest": plan.protected_digest, + } + with sqlite3.connect(tmp_path / "source.db") as conn: for table in ( "raw_authority_censuses", "raw_authority_plans", @@ -86,41 +130,70 @@ def validate(manifest: Path, tier: object, *, connection: sqlite3.Connection) -> "raw_authority_census_plans", "raw_authority_census_post_plans", ): - assert conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0, table - # Accepted state preserved. - row = conn.execute("SELECT revision_authority FROM raw_sessions WHERE raw_id='r-keep'").fetchone() - assert row == ("byte_proven",) + assert conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone() == (0,), table + assert conn.execute("SELECT revision_authority FROM raw_sessions WHERE raw_id='r-keep'").fetchone() == ( + "byte_proven", + ) + assert conn.execute("SELECT * FROM raw_authority_parser_census").fetchall() == before_parser + + receipt = json.loads(report.receipt_path.read_text(encoding="utf-8")) # type: ignore[union-attr] + assert receipt["operation_id"] == plan.operation_id + assert receipt["before_counts"] == plan.before_counts + assert receipt["after_counts"] == dict.fromkeys(plan.before_counts, 0) + assert ( + receipt["receipt_sha256"] + == hashlib.sha256( + json.dumps( + {key: value for key, value in receipt.items() if key != "receipt_sha256"}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode() + ).hexdigest() + ) + + repeated = apply_raw_authority_recovery(plan) + assert repeated.status == "already_satisfied" + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_authority_parser_census SET detail = 'changed' WHERE raw_id = 'r-keep'") + with pytest.raises(RawAuthorityRecoveryError, match="changed"): + apply_raw_authority_recovery(plan) -def test_reset_refuses_to_delete_without_verified_backup_manifest(tmp_path: Path) -> None: +def test_census_reset_dry_run_does_not_mutate_and_apply_requires_backup(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _seed_ledger(tmp_path / "source.db") - - with pytest.raises(ValueError, match="verified source backup manifest"): + _seed_raw(tmp_path / "source.db", "r-keep") + before = (tmp_path / "source.db").read_bytes() + reset_raw_authority_census(tmp_path, dry_run=True) + assert (tmp_path / "source.db").read_bytes() == before + with pytest.raises(RawAuthorityRecoveryError, match="backup authority"): reset_raw_authority_census(tmp_path, dry_run=False) + assert (tmp_path / "source.db").read_bytes() == before + +def test_census_reset_refuses_malformed_ledger(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") with sqlite3.connect(tmp_path / "source.db") as conn: - assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + conn.execute("PRAGMA ignore_check_constraints = ON") + conn.execute("UPDATE raw_authority_censuses SET scope_json = 'not-json'") + with pytest.raises(RawAuthorityRecoveryError, match="malformed"): + inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS) -def test_prune_orphaned_index_revision_seeds(tmp_path: Path) -> None: - initialize_active_archive_root(tmp_path) - source_db = tmp_path / "source.db" - index_db = tmp_path / "index.db" - # One present raw in source; the seeds referencing 'r-gone' are orphaned. - with sqlite3.connect(source_db) as conn: - conn.execute( - "INSERT INTO raw_sessions (raw_id, origin, source_path, source_index, blob_hash, blob_size, " - "acquired_at_ms) VALUES ('r-present','codex-session','/p',0,?,10,1)", - (b"\x02" * 32,), - ) +def _seed_index_seeds(root: Path) -> Path: + source_db = root / "source.db" + index_db = root / "index.db" + _seed_raw(source_db, "r-present") with sqlite3.connect(index_db) as conn: for raw_id in ("r-present", "r-gone"): conn.execute( "INSERT INTO raw_revision_heads (logical_source_key, session_id, accepted_raw_id, " "accepted_source_revision, accepted_content_hash, accepted_frontier_kind, accepted_frontier, " "acquisition_generation, decided_at_ms) VALUES (?,?,?,'sr',?,'byte',1,0,1)", - (f"k-{raw_id}", f"s-{raw_id}", raw_id, b"\x03" * 32), + (f"k-{raw_id}", f"s-{raw_id}", raw_id, bytes.fromhex("03" * 32)), ) conn.execute( "INSERT INTO raw_revision_applications (decision_id, raw_id, session_id, logical_source_key, " @@ -128,20 +201,62 @@ def test_prune_orphaned_index_revision_seeds(tmp_path: Path) -> None: "VALUES (?,?,?,?,'sr',0,'selected_baseline','d',1)", (f"d-{raw_id}", raw_id, f"s-{raw_id}", f"k-{raw_id}"), ) - - active_index = tmp_path / "active-generation" / "index.db" + active_index = root / "active-generation" / "index.db" active_index.parent.mkdir() shutil.copy2(index_db, active_index) - (tmp_path / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + (root / ".index-active-pointer").write_text(str(active_index), encoding="utf-8") + return active_index - dry = prune_orphaned_index_revision_seeds(tmp_path, dry_run=True) - assert (dry.revision_heads, dry.revision_applications, dry.applied) == (1, 1, False) - report = prune_orphaned_index_revision_seeds(tmp_path, dry_run=False) - assert report.applied is True and report.revision_heads == 1 and report.revision_applications == 1 +def test_index_prune_reproduces_orphan_failure_and_preserves_present_rows( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + initialize_active_archive_root(tmp_path) + active_index = _seed_index_seeds(tmp_path) + backup = _backup_authority(tmp_path, monkeypatch, tier="index") + + def bypass(*_args: object, **_kwargs: object) -> None: + raise AssertionError("direct storage prune bypass was called") + + monkeypatch.setattr("polylogue.storage.raw_authority.prune_orphaned_index_revision_seeds", bypass) + dry = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS) + assert dry.before_counts == {"raw_revision_heads": 2, "raw_revision_applications": 2} + assert sqlite3.connect(active_index).execute("SELECT COUNT(*) FROM raw_revision_heads").fetchone() == (2,) + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) + report = apply_raw_authority_recovery(plan) + assert report.status == "applied" + assert report.after_counts == {"raw_revision_heads": 1, "raw_revision_applications": 1} with sqlite3.connect(active_index) as conn: - assert {r[0] for r in conn.execute("SELECT accepted_raw_id FROM raw_revision_heads")} == {"r-present"} - assert {r[0] for r in conn.execute("SELECT raw_id FROM raw_revision_applications")} == {"r-present"} - with sqlite3.connect(index_db) as conn: - assert {r[0] for r in conn.execute("SELECT accepted_raw_id FROM raw_revision_heads")} == {"r-present", "r-gone"} + assert {row[0] for row in conn.execute("SELECT accepted_raw_id FROM raw_revision_heads")} == {"r-present"} + assert {row[0] for row in conn.execute("SELECT raw_id FROM raw_revision_applications")} == {"r-present"} + with sqlite3.connect(tmp_path / "index.db") as conn: + assert {row[0] for row in conn.execute("SELECT accepted_raw_id FROM raw_revision_heads")} == { + "r-present", + "r-gone", + } + assert apply_raw_authority_recovery(plan).status == "already_satisfied" + + +def test_index_prune_refuses_stale_active_pointer_and_wrong_backup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + initialize_active_archive_root(tmp_path) + _seed_index_seeds(tmp_path) + backup = _backup_authority(tmp_path, monkeypatch, tier="index") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) + other = tmp_path / "other" / "index.db" + other.parent.mkdir() + shutil.copy2(tmp_path / "index.db", other) + (tmp_path / ".index-active-pointer").write_text(str(other), encoding="utf-8") + with pytest.raises(RawAuthorityRecoveryError, match="stale|changed"): + apply_raw_authority_recovery(plan) + with pytest.raises(RawAuthorityRecoveryError, match="does not match"): + apply_raw_authority_recovery(plan, backup_manifest=tmp_path / "different.json") + + +def test_named_compatibility_facade_requires_index_backup(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _seed_index_seeds(tmp_path) + with pytest.raises(RawAuthorityRecoveryError, match="backup authority"): + prune_orphaned_index_revision_seeds(tmp_path, dry_run=False) diff --git a/tests/unit/operations/test_mutation_census.py b/tests/unit/operations/test_mutation_census.py index 590746f4e0..7e90906b53 100644 --- a/tests/unit/operations/test_mutation_census.py +++ b/tests/unit/operations/test_mutation_census.py @@ -69,3 +69,19 @@ def test_derived_maintenance_facade_routes_are_executor_routed() -> None: for operation in ("rebuild_index", "update_index", "rebuild_insights"): assert rows[operation]["status"] == "executor-routed" assert rows[operation]["actuator"].startswith("polylogue.operations.mutation_actuators.") + + +def test_raw_authority_recovery_routes_are_executor_routed() -> None: + rows = {row["operation"]: row for row in _load_rows()} + for operation, actuator in ( + ( + "mutate-reset-raw-authority-census", + "polylogue.maintenance.raw_authority_recovery.ResetRawAuthorityCensusActuator", + ), + ( + "mutate-prune-orphaned-index-revision-seeds", + "polylogue.maintenance.raw_authority_recovery.PruneOrphanedIndexRevisionSeedsActuator", + ), + ): + assert rows[operation]["status"] == "executor-routed" + assert rows[operation]["actuator"] == actuator diff --git a/tests/unit/operations/test_specs.py b/tests/unit/operations/test_specs.py index 1c89f0961b..9d2427d447 100644 --- a/tests/unit/operations/test_specs.py +++ b/tests/unit/operations/test_specs.py @@ -54,6 +54,8 @@ def test_runtime_operation_catalog_covers_the_current_runtime_paths() -> None: "mutate-update-index", "mutate-rebuild-insights", "mutate-resolve-raw-authority-blocker", + "mutate-reset-raw-authority-census", + "mutate-prune-orphaned-index-revision-seeds", "mutate-save-saved-view", "mutate-delete-saved-view", "mutate-save-recall-pack", From e735c17a14f844c89d5e2eb37e11c8cdb15e35e3 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 5 Aug 2026 07:24:32 +0200 Subject: [PATCH 02/27] refactor(cli): remove manual raw frontier application Problem: raw-authority-frontier exposed exact-plan application flags even though daemon convergence already selects executable plans under its write coordinator. What changed: remove the CLI apply, preview, and confirmation inputs while retaining census inspection. Update the maintenance descriptions and daemon documentation, and cover both Click option rejection and the daemon executable-only bounded selection path. Alternatives rejected: retaining the apply flags as a break-glass alias would preserve a redundant manual actuator surface. Compatibility/migration: operators can continue inspecting the recorded census; plan application remains daemon-owned. --- docs/daemon.md | 4 +- .../cli/commands/maintenance/__init__.py | 2 +- .../cli/commands/maintenance/_raw_identity.py | 30 +---------- .../unit/cli/test_archive_maintenance_cli.py | 31 +++++++++--- tests/unit/daemon/test_daemon_cli.py | 50 +++++++++++++++++++ 5 files changed, 77 insertions(+), 40 deletions(-) diff --git a/docs/daemon.md b/docs/daemon.md index 6717deb33c..9af6adc36e 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -24,9 +24,7 @@ polylogued status Raw-evidence authority is an ordinary daemon invariant. After bounded raw materialization, the daemon records one complete accepted-frontier census, applies only byte/provenance-safe plans, and leaves conflicts or missing bytes -as durable remediation references in status. Operators can inspect the same -ledger with `polylogue ops maintenance raw-authority-frontier`; its apply -options are break-glass controls for exact plan IDs, not routine maintenance. +as durable remediation references in status. Operators can inspect and record the same census, without applying plans, with `polylogue ops maintenance raw-authority-frontier`. ## Auto-Discovery diff --git a/polylogue/cli/commands/maintenance/__init__.py b/polylogue/cli/commands/maintenance/__init__.py index a09b7eb6e2..b5625f2760 100644 --- a/polylogue/cli/commands/maintenance/__init__.py +++ b/polylogue/cli/commands/maintenance/__init__.py @@ -75,7 +75,7 @@ "raw-authority-frontier", "_raw_identity", "raw_authority_frontier_command", - "Inspect the complete raw-authority frontier; apply is break-glass only.", + "Inspect and record the raw-authority frontier; plan application is daemon-owned.", ), ( "raw-authority-census", diff --git a/polylogue/cli/commands/maintenance/_raw_identity.py b/polylogue/cli/commands/maintenance/_raw_identity.py index bfcdb4b246..971a2b716c 100644 --- a/polylogue/cli/commands/maintenance/_raw_identity.py +++ b/polylogue/cli/commands/maintenance/_raw_identity.py @@ -11,9 +11,6 @@ @click.command("raw-authority-frontier") -@click.option("--apply-plan", "plan_ids", multiple=True, help="Exact immutable plan id; repeatable.") -@click.option("--preview-census", default=None, help="Completed dry-run census authorizing --apply-plan.") -@click.option("--yes", "confirmed", is_flag=True, help="Confirm the selected break-glass application.") @click.option( "--output-format", type=click.Choice(["plain", "json"]), @@ -23,28 +20,11 @@ @click.pass_obj def raw_authority_frontier_command( env: AppEnv, - plan_ids: tuple[str, ...], - preview_census: str | None, - confirmed: bool, output_format: str, ) -> None: - """Inspect the complete frontier or apply exact plans as break-glass work.""" - config = env.config + """Inspect and record the raw-authority frontier without applying plans.""" try: - if plan_ids: - if not confirmed: - raise click.ClickException("refusing raw-authority application without --yes") - if preview_census is None: - raise click.ClickException("--apply-plan requires --preview-census") - payload = raw_authority.apply_frontier( - config, - preview_census_id=preview_census, - selected_plan_ids=plan_ids, - ).to_dict() - else: - if preview_census is not None or confirmed: - raise click.ClickException("apply options require at least one --apply-plan") - payload = raw_authority.inspect_frontier(config).to_dict() + payload = raw_authority.inspect_frontier(env.config).to_dict() except (FileNotFoundError, KeyError, RuntimeError, ValueError) as exc: if isinstance(exc, click.ClickException): raise @@ -52,12 +32,6 @@ def raw_authority_frontier_command( if output_format == "json": click.echo(json.dumps(payload, indent=2, sort_keys=True)) return - if plan_ids: - click.echo( - f"Applied {payload['executed_plan_count']}/{payload['selected_plan_count']} plan(s); " - f"retryable={payload['retryable_plan_count']} census={payload['census_id']}" - ) - return click.echo( f"Frontier {payload['census_id']}: accepted={payload['accepted_head_count']} " f"plans={payload['plan_count']} executable={payload['executable_plan_count']}" diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index d56fdd2f20..fc1c68681d 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -1830,6 +1830,7 @@ def test_archive_maintenance_help_omits_copy_activation_surface(cli_runner: CliR assert removed not in result.output +<<<<<<< HEAD def test_cursor_authority_reconcile_cli_exposes_only_scoped_inputs(cli_runner: CliRunner) -> None: result = cli_runner.invoke( cli, @@ -1910,9 +1911,15 @@ def test_cursor_authority_reconcile_cli_rejects_mixed_or_missing_mode_options( assert "requires" in result.output or "accepts only" in result.output -def test_raw_authority_frontier_cli_replaces_incident_specific_commands( +@pytest.mark.parametrize( + ("option", "value"), + (("--apply-plan", "raw-authority-frontier:" + "a" * 64), ("--preview-census", "census"), ("--yes", None)), +) +def test_raw_authority_frontier_cli_inspects_without_applying_plans( cli_workspace: dict[str, Path], cli_runner: CliRunner, + option: str, + value: str | None, ) -> None: result = cli_runner.invoke( cli, @@ -1948,21 +1955,29 @@ def test_raw_authority_frontier_cli_replaces_incident_specific_commands( ): assert removed not in help_result.output - apply_without_confirmation = cli_runner.invoke( + frontier_help = cli_runner.invoke( + cli, + ["--plain", "ops", "maintenance", "raw-authority-frontier", "--help"], + catch_exceptions=False, + ) + assert frontier_help.exit_code == 0 + assert "Inspect and record the raw-authority frontier without applying plans." in frontier_help.output + for removed in ("--apply-plan", "--preview-census", "--yes"): + assert removed not in frontier_help.output + + rejected = cli_runner.invoke( cli, [ "--plain", "ops", "maintenance", "raw-authority-frontier", - "--apply-plan", - "raw-authority-frontier:" + "a" * 64, - "--preview-census", - payload["census_id"], + option, + *([value] if value is not None else []), ], ) - assert apply_without_confirmation.exit_code == 1 - assert "without --yes" in apply_without_confirmation.output + assert rejected.exit_code == 2 + assert f"No such option {option!r}." in rejected.output def test_raw_authority_frontier_cli_refuses_durable_census_while_daemon_runs( diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 9d92aa33fd..a9ab34a663 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -676,6 +676,56 @@ def fake_repair(*_args: object, **_kwargs: object) -> object: assert not (archive / "writer-mutated").exists() +def test_converge_raw_authority_frontier_applies_only_bounded_executable_plans( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The daemon route selects executable plans before calling the actuator.""" + from polylogue.daemon import cli as daemon_cli + + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + db_path=tmp_path / "index.db", + ) + census = SimpleNamespace( + census_id="census-1", + items=( + SimpleNamespace(plan_id="safe-1", executable=True), + SimpleNamespace(plan_id="blocked-1", executable=False), + SimpleNamespace(plan_id="safe-2", executable=True), + ), + ) + apply_calls: list[dict[str, object]] = [] + + def fake_apply( + _config: Config, + *, + preview_census_id: str, + selected_plan_ids: tuple[str, ...], + ) -> SimpleNamespace: + apply_calls.append( + { + "preview_census_id": preview_census_id, + "selected_plan_ids": selected_plan_ids, + } + ) + return SimpleNamespace( + retryable_plan_count=0, + selected_plan_count=len(selected_plan_ids), + executed_plan_count=2, + ) + + monkeypatch.setattr("polylogue.product.raw_authority.inspect_frontier", lambda _config: census) + monkeypatch.setattr("polylogue.product.raw_authority.apply_frontier", fake_apply) + + executed = daemon_cli._converge_raw_authority_frontier(config, limit=1) + + assert executed == 2 + assert apply_calls == [{"preview_census_id": "census-1", "selected_plan_ids": ("safe-1",)}] + + def test_maybe_recommend_bulk_rebuild_silent_below_threshold(monkeypatch: pytest.MonkeyPatch) -> None: """A backlog under both thresholds must not trigger the bulk-rebuild recommendation: this exercises the real threshold predicate From b5f7a618dcf3f574e746c96341dd2fd5dbf2d550 Mon Sep 17 00:00:00 2001 From: Sinity Date: Wed, 5 Aug 2026 07:33:34 +0200 Subject: [PATCH 03/27] docs: clarify daemon-owned frontier execution Correct stale references to a manual raw-authority frontier workflow in the cursor evidence, executability verifier, and frontier invariant comment. The daemon remains the only automatic plan-selection and application path described by these surfaces. --- .../verify_raw_authority_frontier_executability.py | 10 +++++----- ...lylogue-xeck9-cursor-authority-census-2026-08-04.md | 2 +- polylogue/storage/raw_reconciler.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/devtools/verify_raw_authority_frontier_executability.py b/devtools/verify_raw_authority_frontier_executability.py index 1d51f03396..e7326d13aa 100644 --- a/devtools/verify_raw_authority_frontier_executability.py +++ b/devtools/verify_raw_authority_frontier_executability.py @@ -8,8 +8,8 @@ each paired with a ``RawAuthorityActuator``. Only actuators with a real ``apply()`` dispatch branch (``_APPLY_DISPATCHED_ACTUATORS``) promise "something automatically executes this"; only states in ``_EXECUTABLE_STATES`` -are ever selected by the daemon or the operator break-glass path -(``item.executable``). polylogue-w32w found a state (``UNRESOLVED_PROVENANCE``) +are ever selected by daemon convergence (``item.executable``). +polylogue-w32w found a state (``UNRESOLVED_PROVENANCE``) paired with a dispatched actuator (``REFINE_QUARANTINE``) that was NOT in ``_EXECUTABLE_STATES`` -- 4,174 blockers demanded an actuator no path could ever select, and the gap accumulated silently for weeks because nothing @@ -200,7 +200,7 @@ def _format_report(report: ExecutabilityReport, *, path: Path) -> str: lines.append("") lines.append( "Frontier states pairing a dispatched actuator with a non-executable " - "state -- no path (daemon or operator) would ever select these:" + "state -- daemon convergence would never select these:" ) for pair in report.violations: lines.append( @@ -208,8 +208,8 @@ def _format_report(report: ExecutabilityReport, *, path: Path) -> str: f"{pair.actuator} has an apply() dispatch branch but {pair.state} is not in _EXECUTABLE_STATES" ) lines.append( - " Fix: either add the state to _EXECUTABLE_STATES (and prove the daemon/operator " - "path can safely select it), or pair this classification with a non-dispatched " + " Fix: either add the state to _EXECUTABLE_STATES (and prove the daemon " + "convergence path can safely select it), or pair this classification with a non-dispatched " "actuator (RawAuthorityActuator.NONE, REACQUIRE, or REQUEST_JUDGMENT)." ) if report.dynamic_sites: diff --git a/docs/evidence/polylogue-xeck9-cursor-authority-census-2026-08-04.md b/docs/evidence/polylogue-xeck9-cursor-authority-census-2026-08-04.md index 548d7b8ced..49f32fdab4 100644 --- a/docs/evidence/polylogue-xeck9-cursor-authority-census-2026-08-04.md +++ b/docs/evidence/polylogue-xeck9-cursor-authority-census-2026-08-04.md @@ -50,7 +50,7 @@ The live cursor is ahead of its accepted full-head frontier. That is production ## Safe reconciliation contract, not performed -There is no cursor-specific dry-run/apply actuator that can safely repair this condition without re-running the real ingest path. Do not use raw-authority frontier application as a shortcut: it is a different break-glass workflow and may persist census observations. The safe sequence for an operator is: +There is no cursor-specific dry-run/apply actuator that can safely repair this condition without re-running the real ingest path. Do not use raw-authority frontier inspection as a cursor repair shortcut: it records census observations, while daemon convergence applies only executable proof-backed plans under its writer coordinator, and neither path reconciles the cursor condition. The safe sequence for an operator is: 1. Stop or confirm quiescence of the daemon, then capture a backup plan and an initial read-only full status receipt: diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index 00d3a0a30d..7e02f6b233 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -158,8 +158,8 @@ def __post_init__(self) -> None: # with an executable state. polylogue-u19l was precisely a # violation of this: REFINE_QUARANTINE (a dispatched actuator) was # being assigned to UNRESOLVED_PROVENANCE (a non-executable state), - # so the daemon and the operator break-glass path could never - # select it -- 4,147 blockers accumulated behind an actuator that + # so daemon convergence could never select it -- 4,147 blockers + # accumulated behind an actuator that # was structurally unreachable through every path that exists. This # makes that exact shape impossible to construct, not merely # undocumented. @@ -167,7 +167,7 @@ def __post_init__(self) -> None: raise ValueError( f"raw-authority frontier item is unreachable: actuator {self.actuator.value!r} has an apply() " f"dispatch branch but state {self.state.value!r} is not in the executability gate " - "(_EXECUTABLE_STATES) -- no path (daemon or operator) would ever select this item for apply " + "(_EXECUTABLE_STATES) -- daemon convergence would never select this item for apply " "(polylogue-u19l/polylogue-w32w)" ) From cf279cd39e244cd36c5cfacc5a6c8612b74f0bee Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 01:31:36 +0200 Subject: [PATCH 04/27] fix(raw-authority): validate frontier apply receipts Problem: product wrappers accepted untyped frontier apply responses, so an actuator could report selected, executed, and retryable counts that did not conserve the selected plans. What changed: type the product frontier wrappers, validate response counts and outcome references, and enforce the same invariant on the durable apply report. Add a regression for an incoherent actuator response. Compatibility/migration: no schema or route change; daemon-owned frontier application remains the only apply path. --- polylogue/product/raw_authority.py | 42 +++++++++++++++++-- polylogue/storage/raw_reconciler.py | 20 +++++++++ .../unit/cli/test_archive_maintenance_cli.py | 1 - tests/unit/product/test_raw_authority.py | 42 +++++++++++++++++++ 4 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 tests/unit/product/test_raw_authority.py diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index 7a86e1c04d..3bc71e6add 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -9,13 +9,14 @@ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast from polylogue.config import Config from polylogue.core.json import JSONDocument if TYPE_CHECKING: from polylogue.sources.revision_backfill import RawParsePrefetchCache + from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport, RawAuthorityFrontierCensus RAW_MATERIALIZATION_ORDINARY_BLOB_LIMIT_BYTES: Final = 64 * 1024 * 1024 @@ -51,20 +52,53 @@ def made_progress(self) -> bool: return self.repaired_sessions > 0 or self.executed_plans > 0 or self.censused_components > 0 -def inspect_frontier(config: Config) -> Any: +def inspect_frontier(config: Config) -> RawAuthorityFrontierCensus: from polylogue.storage.raw_reconciler import inspect_raw_authority_frontier return inspect_raw_authority_frontier(config) -def apply_frontier(config: Config, *, preview_census_id: str, selected_plan_ids: tuple[str, ...]) -> Any: +def _validate_frontier_apply_report(report: object) -> RawAuthorityFrontierApplyReport: + """Reject an actuator response that cannot conserve selected plan outcomes.""" + selected = getattr(report, "selected_plan_count", None) + executed = getattr(report, "executed_plan_count", None) + retryable = getattr(report, "retryable_plan_count", None) + if any(type(count) is not int for count in (selected, executed, retryable)): + raise TypeError("raw authority actuator returned non-integer plan counts") + selected_count = cast(int, selected) + executed_count = cast(int, executed) + retryable_count = cast(int, retryable) + if selected_count <= 0 or executed_count < 0 or retryable_count < 0: + raise ValueError("raw authority actuator returned invalid plan counts") + if executed_count + retryable_count != selected_count: + raise ValueError( + "raw authority actuator returned incoherent plan counts: " + "executed_plan_count + retryable_plan_count must equal selected_plan_count" + ) + outcome_refs = getattr(report, "outcome_refs", None) + if not isinstance(outcome_refs, tuple) or len(outcome_refs) != selected_count: + raise ValueError("raw authority actuator must return one outcome reference per selected plan") + from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport + + if not isinstance(report, RawAuthorityFrontierApplyReport): + raise TypeError("raw authority actuator returned an untyped apply report") + return report + + +def apply_frontier( + config: Config, + *, + preview_census_id: str, + selected_plan_ids: tuple[str, ...], +) -> RawAuthorityFrontierApplyReport: from polylogue.storage.raw_reconciler import apply_raw_authority_frontier - return apply_raw_authority_frontier( + report = apply_raw_authority_frontier( config, preview_census_id=preview_census_id, selected_plan_ids=selected_plan_ids, ) + return _validate_frontier_apply_report(report) def recover_interrupted_frontier(config: Config) -> tuple[str, ...]: diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index 7e02f6b233..6ed77dca4b 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -237,6 +237,26 @@ class RawAuthorityFrontierApplyReport: post_plan_count: int outcome_refs: tuple[str, ...] + def __post_init__(self) -> None: + counts = ( + self.selected_plan_count, + self.executed_plan_count, + self.retryable_plan_count, + ) + if any(type(count) is not int for count in counts): + raise TypeError("raw authority apply report plan counts must be integers") + if self.selected_plan_count <= 0: + raise ValueError("raw authority apply report must contain at least one selected plan") + if self.executed_plan_count < 0 or self.retryable_plan_count < 0: + raise ValueError("raw authority apply report plan counts must be non-negative") + if self.executed_plan_count + self.retryable_plan_count != self.selected_plan_count: + raise ValueError( + "raw authority apply report plan counts are incoherent: " + "executed_plan_count + retryable_plan_count must equal selected_plan_count" + ) + if len(self.outcome_refs) != self.selected_plan_count: + raise ValueError("raw authority apply report must contain one outcome reference per selected plan") + @property def success(self) -> bool: return self.retryable_plan_count == 0 diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index fc1c68681d..d5fb6e99cc 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -1830,7 +1830,6 @@ def test_archive_maintenance_help_omits_copy_activation_surface(cli_runner: CliR assert removed not in result.output -<<<<<<< HEAD def test_cursor_authority_reconcile_cli_exposes_only_scoped_inputs(cli_runner: CliRunner) -> None: result = cli_runner.invoke( cli, diff --git a/tests/unit/product/test_raw_authority.py b/tests/unit/product/test_raw_authority.py new file mode 100644 index 0000000000..cc22862c78 --- /dev/null +++ b/tests/unit/product/test_raw_authority.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from polylogue.config import Config +from polylogue.product import raw_authority + + +def test_apply_frontier_rejects_incoherent_actuator_response( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + db_path=tmp_path / "index.db", + ) + response = SimpleNamespace( + census_id="apply-census-1", + preview_census_id="preview-census-1", + selected_plan_count=1, + executed_plan_count=2, + retryable_plan_count=0, + post_inventory_digest="digest", + post_plan_count=0, + outcome_refs=("detail",), + ) + monkeypatch.setattr( + "polylogue.storage.raw_reconciler.apply_raw_authority_frontier", + lambda *_args, **_kwargs: response, + ) + + with pytest.raises(ValueError, match="incoherent plan counts"): + raw_authority.apply_frontier( + config, + preview_census_id="preview-census-1", + selected_plan_ids=("safe-1",), + ) From 19e0368952202e3e1eff548ab80db4a96037cfa6 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 01:31:43 +0200 Subject: [PATCH 05/27] test(daemon): prove frontier selection under writer lease Problem: the frontier selection regression placed the blocked plan after the executable plan, returned an impossible executed count, and called the daemon helper without its coordinator lease. What changed: put the blocked plan first, assert the exact selected plan id and coherent 1/1/0 counts, and run the production convergence helper through the real daemon coordinator while asserting lease ownership in the actuator. Compatibility/migration: manual CLI frontier apply options remain removed; no new operator route is introduced. --- polylogue/daemon/cli.py | 3 ++- tests/unit/daemon/test_daemon_cli.py | 34 ++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/polylogue/daemon/cli.py b/polylogue/daemon/cli.py index ac744a180c..753bab6703 100644 --- a/polylogue/daemon/cli.py +++ b/polylogue/daemon/cli.py @@ -76,6 +76,7 @@ from polylogue.version import POLYLOGUE_VERSION if TYPE_CHECKING: + from polylogue.config import Config from polylogue.daemon.lifecycle import DaemonLifecycle from polylogue.daemon.parse_prefetch import DaemonParseStage from polylogue.product.raw_authority import RawMaterializationCounts @@ -1531,7 +1532,7 @@ def _browser_capture_spool_has_pending_files() -> bool: return False -def _converge_raw_authority_frontier(config: Any, *, limit: int) -> int: +def _converge_raw_authority_frontier(config: Config, *, limit: int) -> int: """Census the entire accepted frontier and execute a bounded safe slice. This runs only beneath ``DaemonWriteCoordinator``. Conflicts, missing diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index a9ab34a663..471d092cc8 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -680,8 +680,10 @@ def test_converge_raw_authority_frontier_applies_only_bounded_executable_plans( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - """The daemon route selects executable plans before calling the actuator.""" + """The daemon selects executable plans under its real writer lease.""" from polylogue.daemon import cli as daemon_cli + from polylogue.daemon.write_coordinator import daemon_write_coordinator, daemon_write_lease_active + from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport config = Config( archive_root=tmp_path, @@ -692,8 +694,8 @@ def test_converge_raw_authority_frontier_applies_only_bounded_executable_plans( census = SimpleNamespace( census_id="census-1", items=( - SimpleNamespace(plan_id="safe-1", executable=True), SimpleNamespace(plan_id="blocked-1", executable=False), + SimpleNamespace(plan_id="safe-1", executable=True), SimpleNamespace(plan_id="safe-2", executable=True), ), ) @@ -704,25 +706,39 @@ def fake_apply( *, preview_census_id: str, selected_plan_ids: tuple[str, ...], - ) -> SimpleNamespace: + ) -> RawAuthorityFrontierApplyReport: + assert daemon_write_lease_active() apply_calls.append( { "preview_census_id": preview_census_id, "selected_plan_ids": selected_plan_ids, } ) - return SimpleNamespace( - retryable_plan_count=0, + return RawAuthorityFrontierApplyReport( + census_id="apply-census-1", + preview_census_id=preview_census_id, selected_plan_count=len(selected_plan_ids), - executed_plan_count=2, + executed_plan_count=1, + retryable_plan_count=0, + post_inventory_digest="digest", + post_plan_count=2, + outcome_refs=("detail",), ) monkeypatch.setattr("polylogue.product.raw_authority.inspect_frontier", lambda _config: census) - monkeypatch.setattr("polylogue.product.raw_authority.apply_frontier", fake_apply) + monkeypatch.setattr("polylogue.storage.raw_reconciler.apply_raw_authority_frontier", fake_apply) + + async def run_under_daemon_coordinator() -> int: + return await daemon_write_coordinator().run_sync( + "maintenance.raw_authority_frontier", + daemon_cli._converge_raw_authority_frontier, + config, + limit=1, + ) - executed = daemon_cli._converge_raw_authority_frontier(config, limit=1) + executed = asyncio.run(run_under_daemon_coordinator()) - assert executed == 2 + assert executed == 1 assert apply_calls == [{"preview_census_id": "census-1", "selected_plan_ids": ("safe-1",)}] From cef354e90d56fac7eb253c276aaccf6ec06956ee Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 01:38:21 +0200 Subject: [PATCH 06/27] fix(raw-authority): share frontier receipt validation Problem: count conservation was independently implemented in the storage report constructor and product wrapper, allowing the two correctness authorities to drift. What changed: extract one storage-side validator used by both boundaries, then cover incoherent report construction and product rejection of a coherent untyped response separately. Adversarial review: iteration 1 identified and this commit removes the duplicated validation logic. --- polylogue/product/raw_authority.py | 28 ++---------- polylogue/storage/raw_reconciler.py | 55 ++++++++++++++++++------ tests/unit/product/test_raw_authority.py | 44 +++++++++++++++++++ 3 files changed, 89 insertions(+), 38 deletions(-) diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index 3bc71e6add..a4bc861852 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final from polylogue.config import Config from polylogue.core.json import JSONDocument @@ -60,29 +60,9 @@ def inspect_frontier(config: Config) -> RawAuthorityFrontierCensus: def _validate_frontier_apply_report(report: object) -> RawAuthorityFrontierApplyReport: """Reject an actuator response that cannot conserve selected plan outcomes.""" - selected = getattr(report, "selected_plan_count", None) - executed = getattr(report, "executed_plan_count", None) - retryable = getattr(report, "retryable_plan_count", None) - if any(type(count) is not int for count in (selected, executed, retryable)): - raise TypeError("raw authority actuator returned non-integer plan counts") - selected_count = cast(int, selected) - executed_count = cast(int, executed) - retryable_count = cast(int, retryable) - if selected_count <= 0 or executed_count < 0 or retryable_count < 0: - raise ValueError("raw authority actuator returned invalid plan counts") - if executed_count + retryable_count != selected_count: - raise ValueError( - "raw authority actuator returned incoherent plan counts: " - "executed_plan_count + retryable_plan_count must equal selected_plan_count" - ) - outcome_refs = getattr(report, "outcome_refs", None) - if not isinstance(outcome_refs, tuple) or len(outcome_refs) != selected_count: - raise ValueError("raw authority actuator must return one outcome reference per selected plan") - from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport - - if not isinstance(report, RawAuthorityFrontierApplyReport): - raise TypeError("raw authority actuator returned an untyped apply report") - return report + from polylogue.storage.raw_reconciler import validate_raw_authority_frontier_apply_report + + return validate_raw_authority_frontier_apply_report(report) def apply_frontier( diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index 6ed77dca4b..63b232db30 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -238,24 +238,12 @@ class RawAuthorityFrontierApplyReport: outcome_refs: tuple[str, ...] def __post_init__(self) -> None: - counts = ( + _validate_raw_authority_frontier_apply_counts( self.selected_plan_count, self.executed_plan_count, self.retryable_plan_count, + self.outcome_refs, ) - if any(type(count) is not int for count in counts): - raise TypeError("raw authority apply report plan counts must be integers") - if self.selected_plan_count <= 0: - raise ValueError("raw authority apply report must contain at least one selected plan") - if self.executed_plan_count < 0 or self.retryable_plan_count < 0: - raise ValueError("raw authority apply report plan counts must be non-negative") - if self.executed_plan_count + self.retryable_plan_count != self.selected_plan_count: - raise ValueError( - "raw authority apply report plan counts are incoherent: " - "executed_plan_count + retryable_plan_count must equal selected_plan_count" - ) - if len(self.outcome_refs) != self.selected_plan_count: - raise ValueError("raw authority apply report must contain one outcome reference per selected plan") @property def success(self) -> bool: @@ -265,6 +253,44 @@ def to_dict(self) -> JSONDocument: return json_document(dataclasses.asdict(self) | {"success": self.success}) +def _validate_raw_authority_frontier_apply_counts( + selected: object, + executed: object, + retryable: object, + outcome_refs: object, +) -> None: + counts = (selected, executed, retryable) + if any(type(count) is not int for count in counts): + raise TypeError("raw authority apply report plan counts must be integers") + selected_count = cast(int, selected) + executed_count = cast(int, executed) + retryable_count = cast(int, retryable) + if selected_count <= 0: + raise ValueError("raw authority apply report must contain at least one selected plan") + if executed_count < 0 or retryable_count < 0: + raise ValueError("raw authority apply report plan counts must be non-negative") + if executed_count + retryable_count != selected_count: + raise ValueError( + "raw authority apply report has incoherent plan counts: " + "executed_plan_count + retryable_plan_count must equal selected_plan_count" + ) + if not isinstance(outcome_refs, tuple) or len(outcome_refs) != selected_count: + raise ValueError("raw authority apply report must contain one outcome reference per selected plan") + + +def validate_raw_authority_frontier_apply_report(report: object) -> RawAuthorityFrontierApplyReport: + """Validate and type-check one raw-authority apply response at a boundary.""" + _validate_raw_authority_frontier_apply_counts( + getattr(report, "selected_plan_count", None), + getattr(report, "executed_plan_count", None), + getattr(report, "retryable_plan_count", None), + getattr(report, "outcome_refs", None), + ) + if not isinstance(report, RawAuthorityFrontierApplyReport): + raise TypeError("raw authority actuator returned an untyped apply report") + return report + + def _archive_root(config: Config) -> Path: """Return the archive file-set root housing the currently active database. @@ -1743,4 +1769,5 @@ def recover_interrupted_raw_authority_frontier(config: Config) -> tuple[str, ... "apply_raw_authority_frontier", "inspect_raw_authority_frontier", "recover_interrupted_raw_authority_frontier", + "validate_raw_authority_frontier_apply_report", ] diff --git a/tests/unit/product/test_raw_authority.py b/tests/unit/product/test_raw_authority.py index cc22862c78..e3c337764e 100644 --- a/tests/unit/product/test_raw_authority.py +++ b/tests/unit/product/test_raw_authority.py @@ -7,6 +7,7 @@ from polylogue.config import Config from polylogue.product import raw_authority +from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport def test_apply_frontier_rejects_incoherent_actuator_response( @@ -40,3 +41,46 @@ def test_apply_frontier_rejects_incoherent_actuator_response( preview_census_id="preview-census-1", selected_plan_ids=("safe-1",), ) + + +def test_apply_frontier_rejects_untyped_actuator_response( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + db_path=tmp_path / "index.db", + ) + response = SimpleNamespace( + selected_plan_count=1, + executed_plan_count=1, + retryable_plan_count=0, + outcome_refs=("detail",), + ) + monkeypatch.setattr( + "polylogue.storage.raw_reconciler.apply_raw_authority_frontier", + lambda *_args, **_kwargs: response, + ) + + with pytest.raises(TypeError, match="untyped apply report"): + raw_authority.apply_frontier( + config, + preview_census_id="preview-census-1", + selected_plan_ids=("safe-1",), + ) + + +def test_frontier_apply_report_rejects_incoherent_counts() -> None: + with pytest.raises(ValueError, match="incoherent plan counts"): + RawAuthorityFrontierApplyReport( + census_id="apply-census-1", + preview_census_id="preview-census-1", + selected_plan_count=1, + executed_plan_count=2, + retryable_plan_count=0, + post_inventory_digest="digest", + post_plan_count=0, + outcome_refs=("detail",), + ) From ccb7376553f39955122de7c6c3353695f436f235 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 01:54:02 +0200 Subject: [PATCH 07/27] fix(raw-authority): bind actuator receipts to requests Problem: a typed coherent actuator receipt could still report a different selected-plan count or preview census than the request passed through the product boundary. What changed: bind validation to the exact selected plan tuple length and preview census id, with regressions for both mismatches. Adversarial review: iteration 2 identified the missing request binding and this commit closes it. --- polylogue/product/raw_authority.py | 19 +++++++++-- polylogue/storage/raw_reconciler.py | 17 +++++++++- tests/unit/product/test_raw_authority.py | 43 ++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index a4bc861852..bd4f26b53c 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -58,11 +58,20 @@ def inspect_frontier(config: Config) -> RawAuthorityFrontierCensus: return inspect_raw_authority_frontier(config) -def _validate_frontier_apply_report(report: object) -> RawAuthorityFrontierApplyReport: +def _validate_frontier_apply_report( + report: object, + *, + selected_plan_ids: tuple[str, ...], + preview_census_id: str, +) -> RawAuthorityFrontierApplyReport: """Reject an actuator response that cannot conserve selected plan outcomes.""" from polylogue.storage.raw_reconciler import validate_raw_authority_frontier_apply_report - return validate_raw_authority_frontier_apply_report(report) + return validate_raw_authority_frontier_apply_report( + report, + selected_plan_ids=selected_plan_ids, + preview_census_id=preview_census_id, + ) def apply_frontier( @@ -78,7 +87,11 @@ def apply_frontier( preview_census_id=preview_census_id, selected_plan_ids=selected_plan_ids, ) - return _validate_frontier_apply_report(report) + return _validate_frontier_apply_report( + report, + selected_plan_ids=selected_plan_ids, + preview_census_id=preview_census_id, + ) def recover_interrupted_frontier(config: Config) -> tuple[str, ...]: diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index 63b232db30..ba030a13a5 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -278,7 +278,12 @@ def _validate_raw_authority_frontier_apply_counts( raise ValueError("raw authority apply report must contain one outcome reference per selected plan") -def validate_raw_authority_frontier_apply_report(report: object) -> RawAuthorityFrontierApplyReport: +def validate_raw_authority_frontier_apply_report( + report: object, + *, + selected_plan_ids: tuple[str, ...], + preview_census_id: str, +) -> RawAuthorityFrontierApplyReport: """Validate and type-check one raw-authority apply response at a boundary.""" _validate_raw_authority_frontier_apply_counts( getattr(report, "selected_plan_count", None), @@ -288,6 +293,16 @@ def validate_raw_authority_frontier_apply_report(report: object) -> RawAuthority ) if not isinstance(report, RawAuthorityFrontierApplyReport): raise TypeError("raw authority actuator returned an untyped apply report") + if report.selected_plan_count != len(selected_plan_ids): + raise ValueError( + "raw authority actuator response selected plan count does not match the request: " + f"response={report.selected_plan_count}, request={len(selected_plan_ids)}" + ) + if report.preview_census_id != preview_census_id: + raise ValueError( + "raw authority actuator response preview census does not match the request: " + f"response={report.preview_census_id!r}, request={preview_census_id!r}" + ) return report diff --git a/tests/unit/product/test_raw_authority.py b/tests/unit/product/test_raw_authority.py index e3c337764e..1453ac4660 100644 --- a/tests/unit/product/test_raw_authority.py +++ b/tests/unit/product/test_raw_authority.py @@ -84,3 +84,46 @@ def test_frontier_apply_report_rejects_incoherent_counts() -> None: post_plan_count=0, outcome_refs=("detail",), ) + + +@pytest.mark.parametrize( + ("selected_plan_ids", "preview_census_id", "message"), + [ + (("safe-1", "safe-2"), "preview-census-1", "selected plan count does not match"), + (("safe-1",), "preview-census-2", "preview census does not match"), + ], +) +def test_apply_frontier_rejects_response_bound_to_a_different_request( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + selected_plan_ids: tuple[str, ...], + preview_census_id: str, + message: str, +) -> None: + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + db_path=tmp_path / "index.db", + ) + response = RawAuthorityFrontierApplyReport( + census_id="apply-census-1", + preview_census_id="preview-census-1", + selected_plan_count=1, + executed_plan_count=1, + retryable_plan_count=0, + post_inventory_digest="digest", + post_plan_count=0, + outcome_refs=("detail",), + ) + monkeypatch.setattr( + "polylogue.storage.raw_reconciler.apply_raw_authority_frontier", + lambda *_args, **_kwargs: response, + ) + + with pytest.raises(ValueError, match=message): + raw_authority.apply_frontier( + config, + preview_census_id=preview_census_id, + selected_plan_ids=selected_plan_ids, + ) From 67776b96b2a1d0a10be02ae7d564caedb66fb7d9 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:06:23 +0200 Subject: [PATCH 08/27] fix(raw-authority): enforce daemon-owned frontier apply Bind actuator outcome references to the selected plan ids and reject product applies without the daemon writer lease. Exercise the real coordinator path with coherent receipts and retain inspection-only CLI behavior. --- polylogue/product/raw_authority.py | 5 ++++ polylogue/storage/raw_reconciler.py | 8 +++++ tests/unit/daemon/test_daemon_cli.py | 3 +- tests/unit/product/test_raw_authority.py | 37 +++++++++++++++++++++--- 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/polylogue/product/raw_authority.py b/polylogue/product/raw_authority.py index bd4f26b53c..a865e6ca7f 100644 --- a/polylogue/product/raw_authority.py +++ b/polylogue/product/raw_authority.py @@ -80,6 +80,11 @@ def apply_frontier( preview_census_id: str, selected_plan_ids: tuple[str, ...], ) -> RawAuthorityFrontierApplyReport: + from polylogue.daemon.write_coordinator import daemon_write_lease_active + + if not daemon_write_lease_active(): + raise RuntimeError("raw authority frontier apply requires the daemon writer lease") + from polylogue.storage.raw_reconciler import apply_raw_authority_frontier report = apply_raw_authority_frontier( diff --git a/polylogue/storage/raw_reconciler.py b/polylogue/storage/raw_reconciler.py index ba030a13a5..0fb12f9c08 100644 --- a/polylogue/storage/raw_reconciler.py +++ b/polylogue/storage/raw_reconciler.py @@ -303,6 +303,14 @@ def validate_raw_authority_frontier_apply_report( "raw authority actuator response preview census does not match the request: " f"response={report.preview_census_id!r}, request={preview_census_id!r}" ) + expected_outcome_refs = tuple( + raw_authority_detail_query_handle(report.census_id, plan_id) for plan_id in selected_plan_ids + ) + if report.outcome_refs != expected_outcome_refs: + raise ValueError( + "raw authority actuator response outcome references do not match the requested plan ids: " + f"response={report.outcome_refs!r}, request={expected_outcome_refs!r}" + ) return report diff --git a/tests/unit/daemon/test_daemon_cli.py b/tests/unit/daemon/test_daemon_cli.py index 471d092cc8..c13cf41a1a 100644 --- a/tests/unit/daemon/test_daemon_cli.py +++ b/tests/unit/daemon/test_daemon_cli.py @@ -683,6 +683,7 @@ def test_converge_raw_authority_frontier_applies_only_bounded_executable_plans( """The daemon selects executable plans under its real writer lease.""" from polylogue.daemon import cli as daemon_cli from polylogue.daemon.write_coordinator import daemon_write_coordinator, daemon_write_lease_active + from polylogue.storage.raw_authority import raw_authority_detail_query_handle from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport config = Config( @@ -722,7 +723,7 @@ def fake_apply( retryable_plan_count=0, post_inventory_digest="digest", post_plan_count=2, - outcome_refs=("detail",), + outcome_refs=(raw_authority_detail_query_handle("apply-census-1", "safe-1"),), ) monkeypatch.setattr("polylogue.product.raw_authority.inspect_frontier", lambda _config: census) diff --git a/tests/unit/product/test_raw_authority.py b/tests/unit/product/test_raw_authority.py index 1453ac4660..4574213629 100644 --- a/tests/unit/product/test_raw_authority.py +++ b/tests/unit/product/test_raw_authority.py @@ -7,6 +7,7 @@ from polylogue.config import Config from polylogue.product import raw_authority +from polylogue.storage.raw_authority import raw_authority_detail_query_handle from polylogue.storage.raw_reconciler import RawAuthorityFrontierApplyReport @@ -14,6 +15,7 @@ def test_apply_frontier_rejects_incoherent_actuator_response( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: + monkeypatch.setattr("polylogue.daemon.write_coordinator.daemon_write_lease_active", lambda: True) config = Config( archive_root=tmp_path, render_root=tmp_path / "render", @@ -47,6 +49,7 @@ def test_apply_frontier_rejects_untyped_actuator_response( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: + monkeypatch.setattr("polylogue.daemon.write_coordinator.daemon_write_lease_active", lambda: True) config = Config( archive_root=tmp_path, render_root=tmp_path / "render", @@ -87,10 +90,11 @@ def test_frontier_apply_report_rejects_incoherent_counts() -> None: @pytest.mark.parametrize( - ("selected_plan_ids", "preview_census_id", "message"), + ("selected_plan_ids", "preview_census_id", "outcome_plan_id", "message"), [ - (("safe-1", "safe-2"), "preview-census-1", "selected plan count does not match"), - (("safe-1",), "preview-census-2", "preview census does not match"), + (("safe-1", "safe-2"), "preview-census-1", "safe-1", "selected plan count does not match"), + (("safe-1",), "preview-census-2", "safe-1", "preview census does not match"), + (("safe-1",), "preview-census-1", "safe-2", "outcome references do not match"), ], ) def test_apply_frontier_rejects_response_bound_to_a_different_request( @@ -98,8 +102,10 @@ def test_apply_frontier_rejects_response_bound_to_a_different_request( tmp_path: Path, selected_plan_ids: tuple[str, ...], preview_census_id: str, + outcome_plan_id: str, message: str, ) -> None: + monkeypatch.setattr("polylogue.daemon.write_coordinator.daemon_write_lease_active", lambda: True) config = Config( archive_root=tmp_path, render_root=tmp_path / "render", @@ -114,7 +120,7 @@ def test_apply_frontier_rejects_response_bound_to_a_different_request( retryable_plan_count=0, post_inventory_digest="digest", post_plan_count=0, - outcome_refs=("detail",), + outcome_refs=(raw_authority_detail_query_handle("apply-census-1", outcome_plan_id),), ) monkeypatch.setattr( "polylogue.storage.raw_reconciler.apply_raw_authority_frontier", @@ -127,3 +133,26 @@ def test_apply_frontier_rejects_response_bound_to_a_different_request( preview_census_id=preview_census_id, selected_plan_ids=selected_plan_ids, ) + + +def test_apply_frontier_requires_daemon_writer_lease( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + config = Config( + archive_root=tmp_path, + render_root=tmp_path / "render", + sources=[], + db_path=tmp_path / "index.db", + ) + monkeypatch.setattr( + "polylogue.storage.raw_reconciler.apply_raw_authority_frontier", + lambda *_args, **_kwargs: pytest.fail("the actuator must not run without the daemon writer lease"), + ) + + with pytest.raises(RuntimeError, match="daemon writer lease"): + raw_authority.apply_frontier( + config, + preview_census_id="preview-census-1", + selected_plan_ids=("safe-1",), + ) From d49cc571597396483ce749ce6b0d2d3ff58e1b9f Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:20:02 +0200 Subject: [PATCH 09/27] test(raw-authority): close frontier ownership gaps Problem: the integrated frontier and recovery routes needed one explicit operator-facing ownership statement and a real test that legacy storage helpers cannot mutate either target set. What changed: document daemon-owned frontier application beside the guarded recovery family, record that boundary in the mutation census plan, and test both direct compatibility helpers fail closed. Compatibility/migration: routine frontier application remains daemon-only; reset_raw_authority_census and prune_orphaned_index_revision_seeds retain the existing inspect/apply recovery command family. --- docs/maintenance.md | 8 ++++++++ docs/plans/mutation-census.yaml | 3 +++ .../maintenance/test_raw_authority_reset.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/docs/maintenance.md b/docs/maintenance.md index 1ea7f11601..60c8aa7632 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -798,6 +798,14 @@ confirm-flag-strength authorization bound to that plan's hash, refusing (`preview_stale`) if the blocker was concurrently resolved between preview and confirm. +### Raw-authority frontier ownership and recovery + +Routine raw-authority frontier application is daemon-owned. The daemon selects +only executable proof-backed plans under the writer coordinator and validates +the typed application receipt. `polylogue ops maintenance raw-authority-frontier` +records an inspection census only; it has no manual plan selector or apply +option. + ### `polylogue ops maintenance raw-authority-recovery` - break-glass ledger recovery This command family is the only operator route for the two callerless raw-authority recovery actuators. It is inspect-only by default. The census reset removes only the five poisoned census-planning tables after a verified source-tier backup. The index-seed prune removes only active-index `raw_revision_heads` and `raw_revision_applications` rows whose source raw is absent. Parser census rows, source raws, blob receipts, and present-source revision rows are outside both target sets. diff --git a/docs/plans/mutation-census.yaml b/docs/plans/mutation-census.yaml index a762421842..5f7bd8e63b 100644 --- a/docs/plans/mutation-census.yaml +++ b/docs/plans/mutation-census.yaml @@ -154,6 +154,9 @@ rows: - polylogue.mcp.server_cutover._dispatch_write (operation=delete_annotation, via delete_annotation) # --- Phase 3: executor-routed (raw-authority blocker resolution) ----------- + # Routine raw-authority frontier application is intentionally not an + # operator mutation row. It is a daemon convergence sub-route under the + # writer coordinator; the operator frontier command is inspection-only. # Tonight-discovered operator gap (2026-07-21/22): resolve_raw_authority_ # blocker had a working CLI adapter (raw-authority-blocker-resolve, its own # --yes gate) but NO census entry and no authorization path shared with any diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 916f604dc8..671de21051 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -260,3 +260,22 @@ def test_named_compatibility_facade_requires_index_backup(tmp_path: Path) -> Non _seed_index_seeds(tmp_path) with pytest.raises(RawAuthorityRecoveryError, match="backup authority"): prune_orphaned_index_revision_seeds(tmp_path, dry_run=False) + + +def test_storage_compatibility_helpers_refuse_direct_mutation(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + _seed_index_seeds(tmp_path) + + from polylogue.storage.raw_authority import ( + prune_orphaned_index_revision_seeds as storage_prune_orphaned_index_revision_seeds, + ) + from polylogue.storage.raw_authority import ( + reset_raw_authority_census_ledger as storage_reset_raw_authority_census_ledger, + ) + + with pytest.raises(RuntimeError, match="direct raw-authority census mutation is disabled"): + storage_reset_raw_authority_census_ledger(tmp_path, backup_manifest=None, dry_run=False) + with pytest.raises(RuntimeError, match="direct orphaned-index-seed mutation is disabled"): + storage_prune_orphaned_index_revision_seeds(tmp_path, dry_run=False) From 84048cc38c5999afb0d4e304f21ac2b0281fefb7 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sun, 9 Aug 2026 02:28:45 +0200 Subject: [PATCH 10/27] test(raw-authority): pin recovery safety refusals Add real recovery tests for a running daemon and a changed source-tier fingerprint. Both cases refuse before deleting ledger rows and preserve the poisoned state for later operator recovery. --- .../maintenance/test_raw_authority_reset.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 671de21051..641bd6df71 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -183,6 +183,37 @@ def test_census_reset_refuses_malformed_ledger(tmp_path: Path) -> None: inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS) +def test_census_reset_refuses_running_daemon_before_mutation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + monkeypatch.setattr("polylogue.maintenance.raw_authority_recovery.running_daemon_pid", lambda _config: 123) + + with pytest.raises(RawAuthorityRecoveryError, match="polylogued is running"): + apply_raw_authority_recovery(plan) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + + +def test_census_reset_refuses_changed_source_fingerprint_before_mutation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE raw_authority_parser_census SET detail = 'changed' WHERE raw_id = 'r-keep'") + + with pytest.raises(RawAuthorityRecoveryError, match="stale"): + apply_raw_authority_recovery(plan) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + + def _seed_index_seeds(root: Path) -> Path: source_db = root / "source.db" index_db = root / "index.db" From 819b190ed0504683aadbcf3005d1ab0c32f4c853 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 02:21:30 +0200 Subject: [PATCH 11/27] chore(beads): define raw authority actuator closure --- .beads/issues.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index d4edc70f9d..8550560882 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -213,7 +213,7 @@ {"_type":"issue","id":"polylogue-tfzw0","title":"storage: hook-event blob_refs born orphaned (del raw_id) - 73,427 refs / ~1.94 GiB unreclaimable and invisible to GC","description":"Structural audit H3 (/realm/data/derived/reports/polylogue-structural-audit-2026-08-03.html). _insert_hook_event (archive_tiers/source_write.py:1176-1177) starts with 'del raw_id' while its caller writes blob_refs(ref_type=raw_payload, ref_id=raw_id) for the payload; nothing joins the ref to raw_hook_events (which has NO blob_hash column - payload lives inline as payload_json, so the blob is a duplicate copy nothing reads back). Live: 73,427/116,149 raw_payload refs resolve to no raw_sessions row; ~69,249 hashes / 1.94 GiB with no live referent, growing since 06-29. blob_gc._still_referenced (blob_gc.py:150-221) is a MEMBERSHIP test on blob_refs so these are retained forever, uncounted. Design fork (hook blobs were retained deliberately in the de-inflation, PR #3265 era): (a) first-class retained ref class: new ref_type hook_payload + blob_hash column on raw_hook_events, GC liveness becomes a per-ref_type JOIN; or (b) declare payload_json the record, delete orphan refs, reclaim 1.94 GiB. Either way: make GC liveness a join not membership, add a standing blob_refs-liveness census metric. Related: polylogue-feu0 (same cross-tier reference class gap).","notes":"2026-08-03 invariant I3 run: reference-liveness violation is not hook-events-only - 1,336 blob_refs rows with ref_type='attachment' have no matching raw_artifacts row either. Strengthens the join-not-membership GC redesign: the census must cover every ref_type. Also fold: invariant I8 found 1 session with drifted sessions.message_count vs actual messages count (projection drift, likely lineage tail-extraction related) - investigate while in the area or split out if unrelated.","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T05:07:18Z","created_by":"Sinity","updated_at":"2026-08-06T19:24:35Z","closed_at":"2026-08-06T19:24:35Z","close_reason":"Closed after current-master audit: PR #3847 landed the unified production hook, attachment, sidecar, and unknown-evidence blob-reference liveness map and focused production-route tests. Remaining live blob reconciliation is owned by the reindex proof graph.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"polylogue-qhk8z","title":"PR #3574 duplicate-chain links trip the pre-existing ActiveByteRevisionChainError membership-census guard","description":"Discovered during polylogue-id4n's fresh triage pass. 2 tests fail:\n\n- tests/unit/sources/test_revision_backfill.py::test_backfill_content_cache_across_pages_reduces_parses_and_matches_uncached_archive\n- tests/unit/storage/test_rebuild_paging_content_order.py::test_rebuild_content_order_paging_dedups_first_time_classification_via_content_cache\n\nBoth fail with:\n\n polylogue.storage.sqlite.archive_tiers.revision_governance.ActiveByteRevisionChainError:\n an active byte-revision chain cannot move to membership governance\n\nRoot cause: a genuine cross-feature interaction between two independent,\nindividually-correct changes.\n\n1. PR #3574 (fix(storage): collapse byte-equal duplicates before revision-chain\n proof) added a \"duplicate\" relation to HistoricalRevisionDecision: two\n byte-identical raws (same content, different acquisition path -- the ordinary\n \"re-exported the same conversation\" shape both failing tests construct)\n now get one classified as representative and the other linked to it via\n predecessor_raw_id/baseline_raw_id, mirroring the representative's verdict.\n This is correct and intentional (fixes 50GB of over-quarantined content on\n the live archive).\n\n2. The pre-existing (#3406, long-standing) membership-census guard in\n revision_governance.py's _replace_full_revision_governance requires that a\n raw being promoted to membership governance have NO other raw pointing at\n it via predecessor_raw_id/baseline_raw_id (\"an active byte-revision chain\n cannot move to membership governance\") -- this guard was written assuming\n only genuine incremental append chains create such links.\n\n#3574 now also creates predecessor/baseline links for the DUPLICATE case, which\nthe membership-census guard was never designed to distinguish from a genuine\nin-progress append chain. A backfill that re-parses a raw touched by this\nguard after #3574's dedup linking now hits ActiveByteRevisionChainError where\nit previously succeeded.\n\nConfirmed via git log -S \"ActiveByteRevisionChainError\" that the guard's own\ncode is unchanged since #3406 -- the trigger is #3574's newly-created links,\nnot the guard itself. Confirmed via git show 31614661f that #3574's own\nverification section did not exercise this specific backfill-then-membership-\ncensus interaction (it ran tests/unit/storage/test_raw_revision_authority.py\nand a `-k \"raw_revision or revision_governance or raw_authority\"` selection,\nwhich apparently does not include these two files).\n\nNeeds design judgment: should the membership-census guard learn to\ndistinguish \"duplicate\" relation links (safe to promote past) from genuine\nincremental chain links (unsafe), or should #3574's duplicate-linking be\nscoped to skip cohorts that would trip this guard? Not attempted as a quick\nfix given the sensitivity of this subsystem (raw-authority correctness,\nquarantine-as-absorbing-state history) -- reproduction is solid, fix\ndirection needs an operator/maintainer decision.\n\nReproduction: devtools test tests/unit/sources/test_revision_backfill.py::test_backfill_content_cache_across_pages_reduces_parses_and_matches_uncached_archive tests/unit/storage/test_rebuild_paging_content_order.py::test_rebuild_content_order_paging_dedups_first_time_classification_via_content_cache","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-08-03T00:08:35Z","created_by":"Sinity","updated_at":"2026-08-03T10:30:25Z","closed_at":"2026-08-03T10:30:25Z","close_reason":"Fixed: PR #3616 (exclude byte-identical duplicates from revision baseline tie-break). Both named regression tests pass.","dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-t73c2","title":"Fix the dogfood loop: polylogue's own archive can't answer 'what did agents do' questions","description":"Polylogue's whole thesis is that the archive answers 'what did agents do'. The fanout-operations report (2026-08-02) had to grep 700MB of raw session/subagent JSONL by hand because polylogue itself could not answer these questions: live index.db is at schema v46 with v53 code deployed, recent days of sessions are not ingested, and the daemon has been deliberately held off mid-merge-train.\n\nThis is the SAME root cause as polylogue-9qnzy (P0, schema-currency gap blocking the planned reindex) -- not a separate bug, a direct consequence of it. This bead exists to make explicit the SECOND reason 9qnzy matters: it's not just blocking a planned reindex, it's actively preventing polylogue from dogfooding its own coordination data right now.\n\nOnce 9qnzy resolves and the reindex/daemon-restart sequence completes: make the coordinator dashboard (output-token ratio, dispatch counts, per-lane outcomes, model distribution) a standing polylogue query instead of a bespoke mining pass every time someone wants to know how a fanout session went. Depends on polylogue-9qnzy.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-02T23:40:08Z","created_by":"Sinity","updated_at":"2026-08-02T23:40:08Z","dependencies":[{"issue_id":"polylogue-t73c2","depends_on_id":"polylogue-3bsrp","type":"relates-to","created_at":"2026-08-03T07:01:30Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-t73c2","depends_on_id":"polylogue-9qnzy","type":"blocks","created_at":"2026-08-03T01:40:20Z","created_by":"Sinity","metadata":"{}"},{"issue_id":"polylogue-t73c2","depends_on_id":"polylogue-ltfj9","type":"parent-child","created_at":"2026-08-03T01:40:09Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"polylogue-tw4ar","title":"Raw-authority verdict: persist a cache table + wire daemon convergence (Phase 2 follow-up)","description":"Follow-up from polylogue-w6hql (PR #3593): project_raw_authority_verdicts (polylogue/storage/raw_authority_verdict_projection.py) currently recomputes verdicts on demand by re-running classify_historical_full_revision_streams against live blob storage every call -- correct but not cheap at scale (761K+ census_plans-era cohort sizes). This bead is to design and land a persisted raw_authority_verdicts cache table (additive migration, numbered under storage/sqlite/migrations/source/) plus wiring into DaemonConverger so the cache tracks new/reclassified cohorts without a full rescan each read. Needed before polylogue-ds4b4 item 4 (blob-GC invariant verification) can cheaply check verdicts at scale rather than via the on-demand read path.","design":"DESIGN (2026-08-03): remaining half only — the cache table + invalidation shipped in PR #3628 (migration 024). Build the DaemonConverger warm-keeping stage: a ConvergenceStage in daemon/convergence_stages.py with check (are there cohorts whose logical_source_key changed since their cached cohort_fingerprint, or never-cached cohorts?) and execute (recompute via project_raw_authority_verdicts and upsert the cache in bounded batches). Use false_means_pending to push remaining backlog into convergence_debt rather than blocking; main process is the sole writer — no worker-process computation of the byte-proof classifier. Invalidation is already content-keyed (cohort_fingerprint over (raw_id, revision_kind, blob_hash) rows, storage/raw_authority_verdict_cache.py) — the stage only needs to FIND stale/missing cohorts cheaply (e.g. join raw_sessions cohort fingerprints against cache rows), never trust elapsed time. Pitfall: append-kind cohorts raise NotImplementedError in the projection — the stage must skip them typed-visibly (count reported), not crash, until w6hql stage-3 extends coverage. Consumer readiness: ds4b4 item 4 reads through the cache once this stage keeps it warm.\n","acceptance_criteria":"1. A DaemonConverger stage exists (daemon/convergence_stages.py) that finds never-cached and fingerprint-stale cohorts and upserts raw_authority_verdicts in bounded batches, deferring backlog via false_means_pending; unit test through the real stage interface.\n2. Append-kind cohorts are skipped typed-visibly (reported count), not crashed on, until w6hql extends coverage.\n3. A repeated read (e.g. ds4b4-style GC invariant check) hits the cache (no classify_historical_full_revision_streams recompute) — proven by a test asserting call counts or receipts.\n4. Cache staleness is content-keyed only (cohort_fingerprint); no time-based trust. Verify: devtools test -k verdict_cache; devtools test -k convergence.","notes":"2026-08-03: PR #3628 shipped the persisted raw_authority_verdicts cache table + cohort-fingerprint invalidation (SOURCE_SCHEMA_VERSION 24, migration 024). Remaining scope: wiring a DaemonConverger stage to keep the cache warm proactively -- deliberately deferred per that PR's own body. Bead stays open for that remaining half.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-02T22:08:58Z","created_by":"Sinity","updated_at":"2026-08-03T11:09:58Z","dependency_count":0,"dependent_count":3,"comment_count":0} +{"_type":"issue","id":"polylogue-tw4ar","title":"Raw-authority verdict: persist a cache table + wire daemon convergence (Phase 2 follow-up)","description":"Follow-up from polylogue-w6hql (PR #3593): project_raw_authority_verdicts (polylogue/storage/raw_authority_verdict_projection.py) currently recomputes verdicts on demand by re-running classify_historical_full_revision_streams against live blob storage every call -- correct but not cheap at scale (761K+ census_plans-era cohort sizes). This bead is to design and land a persisted raw_authority_verdicts cache table (additive migration, numbered under storage/sqlite/migrations/source/) plus wiring into DaemonConverger so the cache tracks new/reclassified cohorts without a full rescan each read. Needed before polylogue-ds4b4 item 4 (blob-GC invariant verification) can cheaply check verdicts at scale rather than via the on-demand read path.","design":"DESIGN (2026-08-03): remaining half only — the cache table + invalidation shipped in PR #3628 (migration 024). Build the DaemonConverger warm-keeping stage: a ConvergenceStage in daemon/convergence_stages.py with check (are there cohorts whose logical_source_key changed since their cached cohort_fingerprint, or never-cached cohorts?) and execute (recompute via project_raw_authority_verdicts and upsert the cache in bounded batches). Use false_means_pending to push remaining backlog into convergence_debt rather than blocking; main process is the sole writer — no worker-process computation of the byte-proof classifier. Invalidation is already content-keyed (cohort_fingerprint over (raw_id, revision_kind, blob_hash) rows, storage/raw_authority_verdict_cache.py) — the stage only needs to FIND stale/missing cohorts cheaply (e.g. join raw_sessions cohort fingerprints against cache rows), never trust elapsed time. Pitfall: append-kind cohorts raise NotImplementedError in the projection — the stage must skip them typed-visibly (count reported), not crash, until w6hql stage-3 extends coverage. Consumer readiness: ds4b4 item 4 reads through the cache once this stage keeps it warm.\n","acceptance_criteria":"1. A DaemonConverger stage exists (daemon/convergence_stages.py) that finds never-cached and fingerprint-stale cohorts and upserts raw_authority_verdicts in bounded batches, deferring backlog via false_means_pending; unit test through the real stage interface.\n2. Append-kind cohorts are skipped typed-visibly (reported count), not crashed on, until w6hql extends coverage.\n3. A repeated read (e.g. ds4b4-style GC invariant check) hits the cache (no classify_historical_full_revision_streams recompute) — proven by a test asserting call counts or receipts.\n4. Cache staleness is content-keyed only (cohort_fingerprint); no time-based trust. Verify: devtools test -k verdict_cache; devtools test -k convergence.","notes":"2026-08-03: PR #3628 shipped the persisted raw_authority_verdicts cache table + cohort-fingerprint invalidation (SOURCE_SCHEMA_VERSION 24, migration 024). Remaining scope: wiring a DaemonConverger stage to keep the cache warm proactively -- deliberately deferred per that PR's own body. Bead stays open for that remaining half.","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-02T22:08:58Z","created_by":"Sinity","updated_at":"2026-08-03T11:09:58Z","dependencies":[{"issue_id":"polylogue-tw4ar","depends_on_id":"polylogue-fbkr","type":"discovered-from","created_at":"2026-08-10T00:18:54Z","created_by":"Sinity","metadata":"{}"}],"dependency_count":0,"dependent_count":3,"comment_count":0} {"_type":"issue","id":"polylogue-gysk3","title":"message_identity_hash reads position-derived provider_message_id fallback (attachment-class bug, unfixed for messages)","description":"Found during polylogue-ds4b4 item 3 investigation (position-derived synthetic\nidentity audit). polylogue-hith/qkuq already found and fixed exactly this\nclass of bug for attachments: a synthetic id seeded partly by array index\n(`att-\u003chash(message_id:name:index)\u003e`) is unstable across export vintages that\nreorder/insert array entries, causing false divergence in revision-authority\nmembership comparison. The fix there was NOT to remove the synthetic\ngenerator (still used as a last-resort storage/display id, seed no longer\nincludes index) but to make `attachment_identity_hash`\n(polylogue/pipeline/ids.py:254) stop reading either the real or synthetic\nattachment id at all -- it hashes only (message_id, name, mime_type).\n\nThe message-level sibling, `message_identity_hash` (polylogue/pipeline/ids.py:212),\nhas the analogous doc claim (\"A provider's own message id is stable across\nre-exports even when the export's array ordering is not\") but no such\nexclusion mechanism -- it hashes the message's `id` directly, and that id\nIS `provider_message_id`, which multiple parsers construct as\n`f\"msg-{index}\"`/`f\"{record_type}-{index}\"` when the raw record carries no\nnative id of its own:\n\n - polylogue/sources/parsers/claude/common.py:912-914 -- `f\"msg-{index}\"`\n - polylogue/sources/parsers/claude/code_parser.py:1620 -- `str(record_uuid or f\"msg-{index}\")`\n - polylogue/sources/parsers/codex.py:1592,1895,1931,2010,2200,2385 --\n `f\"function-call-{index}\"`, `f\"function-call-output-{index}\"`,\n `f\"reasoning-{index}\"`, `f\"{record_type}-{index}\"`,\n `f\"compaction-summary-{idx}\"`\n - polylogue/sources/parsers/local_agent.py:205,252 -- `f\"msg-{index}\"`\n - polylogue/sources/parsers/grok.py:139 -- `f\"{fallback_id}:{index}\"` (unconditional, no real id ever present)\n - polylogue/sources/parsers/drive.py:345 -- `f\"chunk-{idx}\"`\n - polylogue/sources/parsers/chatgpt.py:604 -- `f\"msg-{idx}\"`\n - polylogue/sources/parsers/base_support.py:360 -- `f\"msg-{idx}\"` (shared segment-message builder)\n - polylogue/sources/parsers/antigravity.py:414 -- `f\"{cascade_id}:{index}:{_message_kind(heading)}\"`\n\nUnlike attachments, there is no separate field to fall back to for messages\n-- `provider_message_id` IS the sole identity input by construction\n(`message_identity_hash(*, id: str)`'s fixed keyword-only signature), so the\n\"exclude both real and synthetic id\" fix pattern used for attachments\ndoesn't directly transplant. This needs its own design: likely a\ncomparison-identity axis that anchors on structural position (index within\nthe message array) ONLY when no provider-native id exists, combined with a\ncontent-similarity fallback, or an explicit typed\n\"positionally-anchored, not identity-anchored\" marker threaded through\nsession_revision_membership.py's comparison so a reorder is detected as\n\"can't prove sameness\" rather than silently comparing wrong pairs as if\nmessage ids matched.\n\nNot fixed in polylogue-ds4b4's session: this is genuinely new-discovered\ndebt, and reworking a `pipeline/ids.py` core identity function used\narchive-wide is a substantial, high-risk change (touches every provider's\ncontent-hash/revision-membership comparison) that deserves its own\ndedicated, unhurried session with its own regression-test design -- not a\nrushed fix bundled into an unrelated raw-authority-Phase-3 PR. ds4b4's own\nscope was a preventive LINT for this pattern (shipped separately, flags\nfuture occurrences of position-derived identity construction), not fixing\nevery existing instance.\n\nRef polylogue-ds4b4 (raw-authority redesign Phase 3, item 3)","status":"closed","priority":1,"issue_type":"bug","owner":"ezo.dev@gmail.com","created_at":"2026-08-02T19:36:47Z","created_by":"Sinity","updated_at":"2026-08-03T08:14:39Z","closed_at":"2026-08-03T08:14:39Z","close_reason":"Fixed (in-scope instance): PR #3604 (00e40ef30). _message_comparison_id prefers real provider_message_id, falls back to role+timestamp content anchor instead of position index; only falls back to position when neither exists. Red-first test. NOTE: root cause (parsers baking position-derived ids directly into provider_message_id) is tracked separately - all 18 call sites already acked in docs/plans/position-derived-identity-acks.json referencing this bead.","dependency_count":0,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"polylogue-4zqh3","title":"Acquire the hermes-comparison recovery packet: shared-page decode + sole-copy attachment payloads","description":"A recovery bundle now lives at /realm/data/exports/chatlog/raw/recovery/hermes-project-comparison-2026-07/ (moved from /realm/inbox 2026-08-02; README + SHA256SUMS inside). Two capture gaps, verified against the live archive read-only on 2026-08-02: (1) the ChatGPT shared-page decode chatgpt-shared-decode-6a4ac87b/ (conversation 6a4ac87b, title 'Project and Codebase Analysis', 948 messages, decoded from the share-page React Router stream into messages.json + md + raw html) matches NO raw_sessions row by native_id — the session is entirely absent from the archive and the decode format has no parser; (2) the Claude.ai session claude-ai-export:2c2eab57-fc6c-4c61-99fa-f61af3b7ac57 IS acquired+indexed (raw 8e622747..., quarantined) but 70 of its 83 attachment refs are unfetched with 0 bytes, and the actual payload bytes sit only in this packet (hermes-agent-main.zip 57,853,455 B sha256 31267de3..., hermes-agent-all.tar.gz 257,839,520 B sha256 1b4eba44..., full ChatGPT temporary transcript 309,149 B sha256 db526f41... — none of the three hashes exist in the blob store). Wanted: an ingest path for the decode (or a one-off import), and attachment-byte acquisition from local packet files so the unfetched refs become acquired blobs. The packet is the sole copy of these bytes; exclude from any prune.","design":"DESIGN (2026-08-03): two independent acquisitions from the sole-copy recovery packet (/realm/data/exports/chatlog/raw/recovery/hermes-project-comparison-2026-07/; README + SHA256SUMS; EXCLUDE FROM ANY PRUNE — sole copy):\n1. ChatGPT shared-page decode (conversation 6a4ac87b, 948 messages, messages.json + md + raw html): the decode format has no parser. Options: (a) a narrow detector + parser for the decoded messages.json shape at the document-tightness level in sources/dispatch.py (durable: future share-page decodes ingest too); (b) one-off import via a conversion script mapping the decode into an existing admitted shape. Prefer (a) if the messages.json shape is close to chatgpt-export's mapping node structure (likely, it was decoded from the share-page React Router stream); the parser reuses chatgpt.py's message lowering. Origin question: it is a chatgpt conversation — admit as chatgpt-export with acquisition evidence marking the share-page provenance, not a new origin.\n2. Attachment-byte backfill for claude-ai-export:2c2eab57... (70/83 refs unfetched, 0 bytes; the three packet payloads' sha256 absent from the blob store): an acquisition path that matches local packet files to unfetched attachment refs (by declared hash where the export carries one, else by operator-asserted mapping recorded as evidence) and publishes blobs + flips acquisition_status to acquired. Reuses the attachment blob-write path from #2469 (_acquire_attachment_blob/_write_attachments); the raw row is quarantined — attachment acquisition must not depend on the session's authority state.\nBoth feed f1vg's attachment-fidelity and absence buckets; record before/after bucket counts.\n","acceptance_criteria":"1. The shared-page decode conversation (6a4ac87b, 948 messages) is ingested and queryable (as chatgpt-export with share-page provenance evidence), via parser or documented one-off import; raw bytes + provenance recorded in source.db.\n2. The claude-ai session's 70 unfetched attachment refs become acquired blobs with true SHA-256s from the packet payloads (the three named hashes present in the blob store); acquisition does not depend on the raw row's authority state.\n3. f1vg's absence and attachment-fidelity buckets drop accordingly (before/after recorded).\n4. The packet directory is protected from prune/cleanup (noted in its README or the owning inventory).\n5. Verify: devtools test -k chatgpt or -k attachments for new paths; read-only live queries for ref status.","notes":"Footprint: polylogue/sources/dispatch.py, polylogue/sources/parsers/hermes_spans.py, polylogue/sources/parsers/chatgpt.py (recovery-packet ingestion: ChatGPT shared-page decode + hermes sole-copy attachment payloads).","status":"open","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-02T18:27:09Z","created_by":"Sinity","updated_at":"2026-08-03T11:12:52Z","dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"polylogue-rn5jh","title":"Wire blob-reference-replace-from-source + embeddings-rescue into daemon automation (mislabeled as 'needs judgment')","notes":"2026-08-02 correction after operator pushback: sharper findings than the original AC framing.\n\nembeddings-rescue: polylogue-04kl (the bead this command was built for) is CLOSED -- the actual rescue already happened (187,888 vectors recovered from the one specific 2026-07-10 retired tier, real production win, done). The command remains in the CLI as generic '--source \u003cany path\u003e' product surface for a scenario that occurred exactly once and is finished. Revised AC: DELETE this command (not 'automate it') unless investigation finds embeddings-tier retirement is a routine recurring event (check EMBEDDINGS_SCHEMA_VERSION bump history/frequency) that would need it again -- if genuinely recurring, THEN build the registry+automation described below; if it was truly one-time, it should simply be removed as dead product surface once 04kl's specific job is confirmed fully done (528 partial sessions + ~8949 non-fully-rescuable sessions were noted as still needing real API embedding in 04kl's own notes -- confirm those don't still need this exact command before deleting).\n\nblob-reference-replace-from-source: checked live production archive directly (2026-08-02): 'polylogue ops maintenance blob-reference-debt' reports 160,609 references / 104,026 distinct blobs / 0 missing / status=ok. There is currently ZERO active debt for this command to repair -- the acquire-blob/commit-row safety invariants (leases + snapshot reference check, per CLAUDE.md) appear to be holding in practice right now, not just in theory. This changes the framing: this isn't a live ongoing backlog needing automation urgently -- it's a rare/historical-scenario tool sitting at zero. Revised AC: (a) confirm whether missing-blob-ref debt has EVER been nonzero on this archive historically (check gc_generations/blob GC pass history, or absence of evidence either way), (b) if it's genuinely rare/historical, a read-only periodic check (fails loud if debt ever appears, rather than silent accumulation) may be sufficient instead of full automation -- don't over-build automation for a zero-occurrence problem, (c) if investigation finds real recurring instances, THEN wire the deterministic replace-from-source logic into daemon convergence as originally scoped.\n2026-08-02 RESOLVED via PR #3585 (branch feature/refactor/retire-embeddings-rescue-guard-blob-debt).\n\nembeddings-rescue: DELETED as dead product surface, confirmed one-time. No mechanism in the codebase ever preserves a retired embeddings.db (reset --database hard-deletes it; storage/sqlite/archive_tiers/embeddings.py has had 4 EMBEDDINGS_SCHEMA_VERSION bumps since inception, none of which route through a \"retired file\" path) -- the embeddings.db.v2-retired-20260710 file the command read was a one-off manual operator rename during the single 2026-07-10 incident, not routine behavior. polylogue-04kl (the bead this served) is closed with 187,888 vectors recovered live 2026-07-28; its own notes explicitly say the remaining 528 partial + 8,949 non-fully-rescuable sessions \"still need real API embedding (separate from this rescue path)\" -- i.e. this exact command does not serve them. Removed polylogue/cli/commands/maintenance/_embeddings_rescue.py, polylogue/storage/embeddings/rescue.py, CLI registration, 3 test files, and the docs/maintenance.md section.\n\nblob-reference-replace-from-source: KEPT as a manual CLI command; did NOT wire into daemon automation. Live archive check (2026-08-02, /realm/db/polylogue): 0 missing / 160,609 references / 104,026 distinct blobs -- confirms rn5jh's earlier zero reading. But git history shows this class of debt is not purely theoretical: 2026-06-26, a verified production backup found 39,586 missing referenced blobs (PR #2422 \"report missing referenced blob debt\"), diagnosed and repaired same-day via classify/direct-restore/replace-from-source (#2423, #2425, #2426, #2427) -- the exact deterministic function this bead asked about. It has held at 0 for 5+ weeks since that incident. Read as \"rare, real, not currently recurring\" rather than \"never happened\" or \"actively ongoing\" -- built the lightweight always-on detector the notes asked for rather than full automation: added _check_blob_reference_debt_expensive to polylogue/daemon/health.py's EXPENSIVE tier (reuses the same read-only scan_blob_reference_debt scanner `polylogue ops backup` already runs), OK at 0 missing, ERROR otherwise with a pointer to the existing classify command. This closes the actual gap (nothing previously alerted on recurrence outside a manual backup run) without building daemon-convergence wiring for a problem that hasn't recurred in 5+ weeks of live operation.\n\nVerification: devtools test (55 passed, includes 2 new OK/ERROR-path tests + anti-vacuity check that removing the health-check wiring makes test_expensive_tier_inventory_pinned fail); devtools verify --quick exit 0; 2 pre-existing unrelated test_archive_maintenance_cli.py failures confirmed via git stash to reproduce on origin/master.\n\nPR: https://github.com/Sinity/polylogue/pull/3585","status":"closed","priority":1,"issue_type":"task","owner":"ezo.dev@gmail.com","created_at":"2026-08-02T16:41:01Z","created_by":"Sinity","updated_at":"2026-08-02T20:19:19Z","closed_at":"2026-08-02T20:19:19Z","close_reason":"PR #3585 merged: embeddings-rescue confirmed one-time migration (job done, table deleted); blob-reference debt guarded loudly via new EXPENSIVE health-check tier (_check_blob_reference_debt_expensive) reusing scan_blob_reference_debt — fails loud if debt reappears, currently zero live. No daemon automation wiring needed since both were confirmed non-recurring/already-zero rather than needing ongoing automation.","dependency_count":0,"dependent_count":0,"comment_count":0} From fb051b5a0f3dfc4902ec3fd6fc076f38275623a1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 02:43:04 +0200 Subject: [PATCH 12/27] fix(raw-authority): recover receipts after finalization failure Problem: recovery committed its target transaction before writing the only receipt, so a receipt I/O failure left no restartable evidence of the mutation. Receipt destinations also accepted arbitrary paths, while the route was described too broadly as daemon-owned.\n\nWhat changed: persist and fsync an immutable archive-owned intent before mutation, finalize it after commit, and resume that intent under offline ownership after a final receipt failure. Reject receipt paths outside the archive maintenance state and state the stopped-daemon operator-maintenance boundary in the operation registry and docs.\n\nCompatibility: routine raw-authority frontier convergence remains daemon-owned. Break-glass recovery remains an explicit offline operator action. --- docs/daemon.md | 2 + docs/maintenance.md | 2 +- docs/plans/mutation-census.yaml | 2 + .../maintenance/_raw_authority_recovery.py | 2 +- .../maintenance/raw_authority_recovery.py | 206 +++++++++++++++--- polylogue/operations/specs.py | 5 +- .../maintenance/test_raw_authority_reset.py | 51 +++++ 7 files changed, 237 insertions(+), 33 deletions(-) diff --git a/docs/daemon.md b/docs/daemon.md index 9af6adc36e..b97f0e7455 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -26,6 +26,8 @@ materialization, the daemon records one complete accepted-frontier census, applies only byte/provenance-safe plans, and leaves conflicts or missing bytes as durable remediation references in status. Operators can inspect and record the same census, without applying plans, with `polylogue ops maintenance raw-authority-frontier`. +The separate `raw-authority-recovery` command is not part of this daemon-owned convergence route. It is an explicit offline operator-maintenance action: it refuses while `polylogued` is running, takes archive ownership and the rebuild lease, and leaves a restartable receipt trail under the archive root. + ## Auto-Discovery The daemon watches these directories by default: diff --git a/docs/maintenance.md b/docs/maintenance.md index 60c8aa7632..d28b080974 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -826,7 +826,7 @@ polylogue ops maintenance raw-authority-recovery \ --output-format json ``` -Apply refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation, runs one failure-atomic transaction, and writes a self-hashed immutable receipt. It does not invoke the broad index reset or reparse path. +Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent under `/.maintenance-state/raw-authority-recovery/`; a restart finalizes that intent into the self-hashed receipt if the mutation committed. Receipt destinations are restricted to that archive-owned durable location. It does not invoke the broad index reset or reparse path. ### Measuring Codex UUID-title coverage diff --git a/docs/plans/mutation-census.yaml b/docs/plans/mutation-census.yaml index 5f7bd8e63b..50d26543d0 100644 --- a/docs/plans/mutation-census.yaml +++ b/docs/plans/mutation-census.yaml @@ -176,6 +176,7 @@ rows: - operation: mutate-reset-raw-authority-census spec_name: mutate-reset-raw-authority-census status: executor-routed + execution_owner: offline-operator-maintenance actuator: polylogue.maintenance.raw_authority_recovery.ResetRawAuthorityCensusActuator surfaces: [cli] adapters: @@ -184,6 +185,7 @@ rows: - operation: mutate-prune-orphaned-index-revision-seeds spec_name: mutate-prune-orphaned-index-revision-seeds status: executor-routed + execution_owner: offline-operator-maintenance actuator: polylogue.maintenance.raw_authority_recovery.PruneOrphanedIndexRevisionSeedsActuator surfaces: [cli] adapters: diff --git a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py index a80d1ecb49..9e57490af4 100644 --- a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py +++ b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py @@ -42,7 +42,7 @@ "--receipt-file", type=click.Path(dir_okay=False, path_type=Path), default=None, - help="Immutable receipt destination bound into the dry-run plan.", + help="Immutable receipt destination under the archive-owned maintenance-state directory.", ) @click.option("--output-format", type=click.Choice(["plain", "json"]), default="plain", show_default=True) @click.pass_obj diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index a1a0ad9207..f72113eb2b 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -61,6 +61,7 @@ class RecoveryOperation(StrEnum): PLAN_FORMAT = "polylogue.raw-authority-recovery-plan.v1" RECEIPT_FORMAT = "polylogue.raw-authority-recovery-receipt.v1" +INTENT_FORMAT = "polylogue.raw-authority-recovery-intent.v1" RECOVERY_DIRNAME = "raw-authority-recovery" _RESET_TABLES = ( "raw_authority_blockers", @@ -455,6 +456,26 @@ def _default_receipt_path(root: Path, operation_id: str) -> Path: return root / ".maintenance-state" / RECOVERY_DIRNAME / f"{operation_id}.receipt.json" +def _resolve_receipt_path(root: Path, receipt_path: Path) -> Path: + """Keep recovery evidence inside the archive's durable maintenance state.""" + + receipts_root = (root / ".maintenance-state" / RECOVERY_DIRNAME).resolve(strict=False) + candidate = receipt_path.expanduser().resolve(strict=False) + try: + candidate.relative_to(receipts_root) + except ValueError as exc: + raise RawAuthorityRecoveryError( + f"recovery receipt path must be inside the archive-owned durable location {receipts_root}" + ) from exc + if candidate.suffix != ".json": + raise RawAuthorityRecoveryError("recovery receipt path must end in .json") + return candidate + + +def _intent_path(receipt_path: Path) -> Path: + return receipt_path.with_name(f"{receipt_path.name}.intent.json") + + def _read_json(path: Path) -> dict[str, object]: try: payload = json.loads(path.read_text(encoding="utf-8")) @@ -490,6 +511,11 @@ def _write_immutable(path: Path, payload: dict[str, object], *, digest_field: st with temporary.open("rb") as handle: os.fsync(handle.fileno()) os.link(temporary, path) + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) except FileExistsError: existing = _read_json(path) if existing != stamped: @@ -552,7 +578,7 @@ def _build_plan( sqlite3.connect(f"file:{source_db if target is ArchiveTier.SOURCE else index_db}?mode=ro", uri=True) ) as conn: backup_authority = _validate_backup(backup_manifest, tier=target, connection=conn) - receipt = receipt_path or _default_receipt_path(root, operation_id) + receipt = _resolve_receipt_path(root, receipt_path or _default_receipt_path(root, operation_id)) payload: dict[str, object] = { "format": PLAN_FORMAT, "operation_id": operation_id, @@ -570,7 +596,7 @@ def _build_plan( "candidate_keys": {key: list(value) for key, value in candidate_keys.items()}, "protected_digest": protected, "backup_authority": backup_authority, - "receipt_path": str(receipt.resolve(strict=False)), + "receipt_path": str(receipt), } return RawAuthorityRecoveryPlan( operation_id=operation_id, @@ -588,7 +614,7 @@ def _build_plan( candidate_keys=candidate_keys, protected_digest=protected, backup_authority=backup_authority, - receipt_path=str(receipt.resolve(strict=False)), + receipt_path=str(receipt), plan_digest=_digest(payload), ) @@ -641,15 +667,143 @@ def _revalidate_common(plan: RawAuthorityRecoveryPlan, root: Path, location: Arc _same(source_revision_snapshot(root), plan.source_snapshot, "source snapshot") +def _recovery_intent(plan: RawAuthorityRecoveryPlan) -> dict[str, object]: + return { + "format": INTENT_FORMAT, + "operation_id": plan.operation_id, + "operation": plan.operation, + "archive_root": plan.archive_root, + "plan_digest": plan.plan_digest, + "receipt_path": plan.receipt_path, + "before_counts": plan.before_counts, + "candidate_keys": {key: list(value) for key, value in plan.candidate_keys.items()}, + "protected_digest": plan.protected_digest, + } + + +def _write_recovery_intent(plan: RawAuthorityRecoveryPlan) -> Path: + return _write_immutable( + _intent_path(Path(plan.receipt_path)), + _recovery_intent(plan), + digest_field="intent_sha256", + ) + + +def _intent_for_plan(plan: RawAuthorityRecoveryPlan) -> dict[str, object] | None: + path = _intent_path(Path(plan.receipt_path)) + if not path.is_file(): + return None + payload = _read_json(path) + expected = payload.get("intent_sha256") + if not isinstance(expected, str) or expected != _digest( + {key: value for key, value in payload.items() if key != "intent_sha256"} + ): + raise RawAuthorityRecoveryError("existing recovery intent has an invalid self-hash") + if payload != {**_recovery_intent(plan), "intent_sha256": expected}: + raise RawAuthorityRecoveryError("existing recovery intent belongs to another operation or plan") + return payload + + +def _receipt_payload( + plan: RawAuthorityRecoveryPlan, + *, + before_counts: dict[str, int], + after_counts: dict[str, int], + postflight: dict[str, object], +) -> dict[str, object]: + root = Path(plan.archive_root) + return { + "format": RECEIPT_FORMAT, + "operation_id": plan.operation_id, + "operation": plan.operation, + "archive_root": plan.archive_root, + "plan_digest": plan.plan_digest, + "code_sha": plan.code_sha, + "archive_identity": plan.archive_identity, + "schema_versions": plan.schema_versions, + "active_generation": plan.active_generation, + "source_snapshot_before": plan.source_snapshot, + "source_snapshot_after": source_revision_snapshot(root), + "source_fingerprint_before": plan.source_fingerprint, + "index_fingerprint_before": plan.index_fingerprint, + "source_fingerprint_after": _file_fingerprint(root / "source.db"), + "index_fingerprint_after": _file_fingerprint(ArchiveLocation.resolve(root).active_index_path), + "before_counts": before_counts, + "after_counts": after_counts, + "candidate_keys": {key: list(value) for key, value in plan.candidate_keys.items()}, + "backup_authority": plan.backup_authority, + "protected_digest_before": plan.protected_digest, + "protected_digest_after": postflight["protected_digest"], + "postflight": postflight, + } + + +def _write_recovery_receipt( + plan: RawAuthorityRecoveryPlan, + *, + before_counts: dict[str, int], + after_counts: dict[str, int], + postflight: dict[str, object], +) -> Path: + return _write_immutable( + Path(plan.receipt_path), + _receipt_payload(plan, before_counts=before_counts, after_counts=after_counts, postflight=postflight), + digest_field="receipt_sha256", + ) + + +def _committed_postflight(plan: RawAuthorityRecoveryPlan) -> tuple[dict[str, int], dict[str, object]] | None: + """Return postflight evidence only when an intent's exact mutation committed.""" + + root = Path(plan.archive_root) + location = ArchiveLocation.resolve(root) + operation = RecoveryOperation(plan.operation) + excluded = _RESET_TABLES if operation is RecoveryOperation.RESET_CENSUS else _INDEX_TARGETS + database = root / "source.db" if operation is RecoveryOperation.RESET_CENSUS else location.active_index_path + with closing(sqlite3.connect(f"file:{database}?mode=ro", uri=True)) as conn: + conn.row_factory = sqlite3.Row + if operation is RecoveryOperation.RESET_CENSUS: + after_counts = _count_tables(conn, _RESET_TABLES) + expected_after = dict.fromkeys(_RESET_TABLES, 0) + else: + conn.execute("ATTACH DATABASE ? AS src", (str(root / "source.db"),)) + after_counts = _count_tables(conn, _INDEX_TARGETS) + expected_after = {key: plan.before_counts[key] - len(plan.candidate_keys[key]) for key in _INDEX_TARGETS} + if after_counts != expected_after: + return None + postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) + return after_counts, postflight + + def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: root = Path(plan.archive_root) _require_apply_preconditions(root) + _resolve_receipt_path(root, Path(plan.receipt_path)) location = ArchiveLocation.resolve(root) operation = RecoveryOperation(plan.operation) source_db = root / "source.db" index_db = location.active_index_path excluded = _RESET_TABLES if operation is RecoveryOperation.RESET_CENSUS else _INDEX_TARGETS before_counts = dict(plan.before_counts) + if _intent_for_plan(plan) is not None: + committed = _committed_postflight(plan) + if committed is not None: + after_counts, postflight = committed + receipt_path = _write_recovery_receipt( + plan, + before_counts=before_counts, + after_counts=after_counts, + postflight=postflight, + ) + return RawAuthorityRecoveryReport( + plan=plan, + applied=False, + status="already_satisfied", + receipt_path=receipt_path, + after_counts=after_counts, + postflight=postflight, + ) + _write_recovery_intent(plan) if operation is RecoveryOperation.RESET_CENSUS: with closing(sqlite3.connect(source_db)) as conn: conn.row_factory = sqlite3.Row @@ -710,31 +864,12 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: except Exception: conn.rollback() raise - receipt_payload: dict[str, object] = { - "format": RECEIPT_FORMAT, - "operation_id": plan.operation_id, - "operation": plan.operation, - "archive_root": plan.archive_root, - "plan_digest": plan.plan_digest, - "code_sha": plan.code_sha, - "archive_identity": plan.archive_identity, - "schema_versions": plan.schema_versions, - "active_generation": plan.active_generation, - "source_snapshot_before": plan.source_snapshot, - "source_snapshot_after": source_revision_snapshot(root), - "source_fingerprint_before": plan.source_fingerprint, - "index_fingerprint_before": plan.index_fingerprint, - "source_fingerprint_after": _file_fingerprint(root / "source.db"), - "index_fingerprint_after": _file_fingerprint(ArchiveLocation.resolve(root).active_index_path), - "before_counts": before_counts, - "after_counts": after_counts, - "candidate_keys": {key: list(value) for key, value in plan.candidate_keys.items()}, - "backup_authority": plan.backup_authority, - "protected_digest_before": plan.protected_digest, - "protected_digest_after": postflight["protected_digest"], - "postflight": postflight, - } - receipt_path = _write_immutable(Path(plan.receipt_path), receipt_payload, digest_field="receipt_sha256") + receipt_path = _write_recovery_receipt( + plan, + before_counts=before_counts, + after_counts=after_counts, + postflight=postflight, + ) return RawAuthorityRecoveryReport( plan=plan, applied=True, @@ -799,7 +934,7 @@ def apply(self, plan: MutationPlan, args: _RecoveryArgs) -> MutationReceipt: return MutationReceipt( operation=self.operation, plan_hash=args.expected_plan_digest, - status="applied", + status="applied" if report.applied else "already_satisfied", target_refs=( make_target_ref( "source", @@ -863,6 +998,7 @@ def apply_raw_authority_recovery( ) -> RawAuthorityRecoveryReport: """Apply one exact plan through the named actuator lifecycle.""" selected = _load_plan(plan) if isinstance(plan, Path) else plan + _resolve_receipt_path(Path(selected.archive_root), Path(selected.receipt_path)) if backup_manifest is not None and ( selected.backup_authority is None or str(backup_manifest.resolve(strict=False)) != selected.backup_authority.get("manifest_path") @@ -900,6 +1036,18 @@ def apply_raw_authority_recovery( executor = OperationExecutor() try: location = ArchiveLocation.resolve(root) + # A final receipt may be missing after a process crash or I/O failure, + # but the immutable intent was fsynced before the target transaction. + # Resume that exact intent under the same offline ownership boundary; + # a new PREPARE would correctly look stale after a committed mutation. + if _intent_for_plan(selected) is not None: + with OwnedArchiveLocation.acquire( + location, owner_id=f"raw-authority-recovery:{selected.operation_id}" + ) as owned: + current_location = ArchiveLocation.resolve(root) + assert_owns_archive_location(owned, current_location) + with RebuildLease(root): + return _apply_plan(selected) prepared = executor.prepare(actuator, args) if prepared.context.get("recovery_plan_digest") != selected.plan_digest: raise PlanStaleError("recovery plan is stale before lease acquisition") diff --git a/polylogue/operations/specs.py b/polylogue/operations/specs.py index 2107e87ce5..3b65444a33 100644 --- a/polylogue/operations/specs.py +++ b/polylogue/operations/specs.py @@ -950,7 +950,8 @@ def to_dict(self) -> JSONDocumentList: kind=OperationKind.MAINTENANCE, description=( "Last-resort reset of poisoned raw-authority census bookkeeping. The route is dry-run first, " - "requires an exact source-tier backup-attested plan and stopped-daemon maintenance ownership, " + "requires an exact source-tier backup-attested plan and explicit offline operator-maintenance ownership " + "with the daemon stopped, " "and never touches parser census, accepted raws, or blobs." ), consumes=("raw_authority_census_ledger",), @@ -974,7 +975,7 @@ def to_dict(self) -> JSONDocumentList: description=( "Remove only active-index raw revision seed rows whose source raws are absent. The route binds " "the exact active generation, source snapshot, recoverable index backup, and stopped-daemon " - "maintenance ownership; it never performs a broad index reset." + "offline operator-maintenance ownership; it never performs a broad index reset." ), consumes=("raw_revision_heads", "raw_revision_applications", "raw_sessions"), produces=("raw_authority_index_seed_recovery_receipt",), diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 641bd6df71..06b92baffe 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -160,6 +160,57 @@ def bypass(*_args: object, **_kwargs: object) -> None: apply_raw_authority_recovery(plan) +def test_census_reset_preserves_recovery_evidence_when_final_receipt_write_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A committed reset must leave restartable evidence before finalization.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + + from polylogue.maintenance import raw_authority_recovery + + original_write = raw_authority_recovery._write_immutable + + def fail_final_receipt(path: Path, payload: dict[str, object], *, digest_field: str) -> Path: + if digest_field == "receipt_sha256": + raise OSError("injected final receipt write failure") + return original_write(path, payload, digest_field=digest_field) + + monkeypatch.setattr(raw_authority_recovery, "_write_immutable", fail_final_receipt) + with pytest.raises(RawAuthorityRecoveryError, match="injected final receipt write failure"): + apply_raw_authority_recovery(plan) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (0,) + receipt_path = Path(plan.receipt_path) + intent_path = receipt_path.with_name(f"{receipt_path.name}.intent.json") + assert intent_path.is_file() + assert not receipt_path.exists() + + monkeypatch.setattr(raw_authority_recovery, "_write_immutable", original_write) + recovered = apply_raw_authority_recovery(plan) + assert recovered.status == "already_satisfied" + assert recovered.receipt_path == receipt_path + assert receipt_path.is_file() + + +def test_recovery_receipt_path_must_be_owned_by_the_archive(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + + with pytest.raises(RawAuthorityRecoveryError, match="archive-owned durable location"): + inspect_raw_authority_recovery( + tmp_path, + RecoveryOperation.RESET_CENSUS, + receipt_path=tmp_path.parent / "arbitrary-recovery-receipt.json", + ) + + def test_census_reset_dry_run_does_not_mutate_and_apply_requires_backup(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _seed_ledger(tmp_path / "source.db") From b7b7405c81191b9d4c6dd324f966e6b0dd295674 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 02:56:26 +0200 Subject: [PATCH 13/27] fix(raw-authority): resume durable recovery intents Problem: a completed mutation with a failed final receipt could only be resumed from a retained external plan, and an archive receipt route could traverse a symlink.\n\nWhat changed: persist the complete plan in the fsynced intent, add operation-id resume, reject archive symlink traversal, and record the offline durable-intent continuation boundary.\n\nCompatibility: fresh recovery applies remain executor-routed; only a previously authorized durable intent can finalize its receipt offline. --- docs/maintenance.md | 2 +- docs/plans/mutation-census.yaml | 2 + .../maintenance/_raw_authority_recovery.py | 17 +++++-- .../maintenance/raw_authority_recovery.py | 48 ++++++++++++++++++- polylogue/operations/specs.py | 6 ++- .../maintenance/test_raw_authority_reset.py | 34 ++++++++++++- tests/unit/operations/test_mutation_census.py | 2 + 7 files changed, 101 insertions(+), 10 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index d28b080974..7bcc1bc3b7 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -826,7 +826,7 @@ polylogue ops maintenance raw-authority-recovery \ --output-format json ``` -Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent under `/.maintenance-state/raw-authority-recovery/`; a restart finalizes that intent into the self-hashed receipt if the mutation committed. Receipt destinations are restricted to that archive-owned durable location. It does not invoke the broad index reset or reparse path. +Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent, including the complete plan, under `/.maintenance-state/raw-authority-recovery/`; a restart finalizes that intent into the self-hashed receipt if the mutation committed. Receipt destinations are restricted to that archive-owned durable location and reject symlink traversal. If the external plan file was lost after a final-receipt failure, rerun `--apply --operation-id ` to resume the archive-owned intent. It does not invoke the broad index reset or reparse path. ### Measuring Codex UUID-title coverage diff --git a/docs/plans/mutation-census.yaml b/docs/plans/mutation-census.yaml index 50d26543d0..60425b3d70 100644 --- a/docs/plans/mutation-census.yaml +++ b/docs/plans/mutation-census.yaml @@ -177,6 +177,7 @@ rows: spec_name: mutate-reset-raw-authority-census status: executor-routed execution_owner: offline-operator-maintenance + recovery_continuation: offline-durable-intent actuator: polylogue.maintenance.raw_authority_recovery.ResetRawAuthorityCensusActuator surfaces: [cli] adapters: @@ -186,6 +187,7 @@ rows: spec_name: mutate-prune-orphaned-index-revision-seeds status: executor-routed execution_owner: offline-operator-maintenance + recovery_continuation: offline-durable-intent actuator: polylogue.maintenance.raw_authority_recovery.PruneOrphanedIndexRevisionSeedsActuator surfaces: [cli] adapters: diff --git a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py index 9e57490af4..526e68b9c0 100644 --- a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py +++ b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py @@ -14,6 +14,7 @@ RecoveryOperation, apply_raw_authority_recovery, inspect_raw_authority_recovery, + resume_raw_authority_recovery, write_recovery_plan, ) @@ -26,6 +27,7 @@ help="One exact recovery actuator to inspect or apply.", ) @click.option("--apply", "apply_changes", is_flag=True, help="Apply the exact previously inspected plan.") +@click.option("--operation-id", default=None, help="Recovery operation id for an interrupted intent resume.") @click.option( "--plan-file", type=click.Path(dir_okay=False, path_type=Path), @@ -50,6 +52,7 @@ def raw_authority_recovery_command( env: AppEnv, operation: str, apply_changes: bool, + operation_id: str | None, plan_file: Path | None, backup_manifest: Path | None, receipt_file: Path | None, @@ -60,14 +63,22 @@ def raw_authority_recovery_command( try: selected = RecoveryOperation(operation) if apply_changes: - if plan_file is None: - raise click.ClickException("--apply requires --plan-file from a prior inspect") - report = apply_raw_authority_recovery(plan_file, backup_manifest=backup_manifest) + if plan_file is not None: + report = apply_raw_authority_recovery(plan_file, backup_manifest=backup_manifest) + elif operation_id is not None: + report = resume_raw_authority_recovery( + env.config.archive_root, + selected, + operation_id=operation_id, + ) + else: + raise click.ClickException("--apply requires --plan-file or --operation-id for an interrupted intent") payload = report.to_dict() else: plan_obj = inspect_raw_authority_recovery( env.config.archive_root, selected, + operation_id=operation_id, backup_manifest=backup_manifest, receipt_path=receipt_file, ) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index f72113eb2b..051aa1d3e8 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -459,8 +459,9 @@ def _default_receipt_path(root: Path, operation_id: str) -> Path: def _resolve_receipt_path(root: Path, receipt_path: Path) -> Path: """Keep recovery evidence inside the archive's durable maintenance state.""" - receipts_root = (root / ".maintenance-state" / RECOVERY_DIRNAME).resolve(strict=False) - candidate = receipt_path.expanduser().resolve(strict=False) + archive_root = root.expanduser().resolve(strict=False) + receipts_root = archive_root / ".maintenance-state" / RECOVERY_DIRNAME + candidate = Path(os.path.abspath(receipt_path.expanduser())) try: candidate.relative_to(receipts_root) except ValueError as exc: @@ -469,6 +470,11 @@ def _resolve_receipt_path(root: Path, receipt_path: Path) -> Path: ) from exc if candidate.suffix != ".json": raise RawAuthorityRecoveryError("recovery receipt path must end in .json") + current = archive_root + for component in candidate.relative_to(archive_root).parts: + current /= component + if current.is_symlink(): + raise RawAuthorityRecoveryError(f"recovery receipt path must not traverse an archive symlink: {current}") return candidate @@ -678,6 +684,7 @@ def _recovery_intent(plan: RawAuthorityRecoveryPlan) -> dict[str, object]: "before_counts": plan.before_counts, "candidate_keys": {key: list(value) for key, value in plan.candidate_keys.items()}, "protected_digest": plan.protected_digest, + "plan": plan.to_dict(), } @@ -991,6 +998,42 @@ def _validate_existing_receipt(plan: RawAuthorityRecoveryPlan, receipt: dict[str raise RawAuthorityRecoveryError("existing recovery receipt has inconsistent postflight evidence") +def _plan_from_intent( + archive_root: Path, + operation: RecoveryOperation, + *, + operation_id: str, +) -> RawAuthorityRecoveryPlan: + root = archive_root.expanduser().resolve(strict=False) + receipt_path = _resolve_receipt_path(root, _default_receipt_path(root, operation_id)) + intent_path = _intent_path(receipt_path) + if not intent_path.is_file(): + raise RawAuthorityRecoveryError(f"no restartable recovery intent exists for operation {operation_id!r}") + payload = _read_json(intent_path) + serialized_plan = payload.get("plan") + if not isinstance(serialized_plan, dict): + raise RawAuthorityRecoveryError("existing recovery intent does not contain a complete recovery plan") + plan = RawAuthorityRecoveryPlan.from_dict(cast(Mapping[str, object], serialized_plan)) + if plan.archive_root != str(root) or plan.operation != operation.value or plan.operation_id != operation_id: + raise RawAuthorityRecoveryError("existing recovery intent does not match the requested archive operation") + _resolve_receipt_path(root, Path(plan.receipt_path)) + _intent_for_plan(plan) + return plan + + +def resume_raw_authority_recovery( + archive_root: Path, + operation: RecoveryOperation | str, + *, + operation_id: str, +) -> RawAuthorityRecoveryReport: + """Resume a durable intent when the external dry-run plan artifact is unavailable.""" + + selected = RecoveryOperation(operation) + plan = _plan_from_intent(archive_root, selected, operation_id=operation_id) + return apply_raw_authority_recovery(plan) + + def apply_raw_authority_recovery( plan: RawAuthorityRecoveryPlan | Path, *, @@ -1100,5 +1143,6 @@ def apply_raw_authority_recovery( "ResetRawAuthorityCensusActuator", "apply_raw_authority_recovery", "inspect_raw_authority_recovery", + "resume_raw_authority_recovery", "write_recovery_plan", ] diff --git a/polylogue/operations/specs.py b/polylogue/operations/specs.py index 3b65444a33..86891d8967 100644 --- a/polylogue/operations/specs.py +++ b/polylogue/operations/specs.py @@ -952,7 +952,8 @@ def to_dict(self) -> JSONDocumentList: "Last-resort reset of poisoned raw-authority census bookkeeping. The route is dry-run first, " "requires an exact source-tier backup-attested plan and explicit offline operator-maintenance ownership " "with the daemon stopped, " - "and never touches parser census, accepted raws, or blobs." + "and never touches parser census, accepted raws, or blobs. Fresh applies run through OperationExecutor; " + "only a durable already-authorized intent may resume receipt finalization offline." ), consumes=("raw_authority_census_ledger",), produces=("raw_authority_census_recovery_receipt",), @@ -975,7 +976,8 @@ def to_dict(self) -> JSONDocumentList: description=( "Remove only active-index raw revision seed rows whose source raws are absent. The route binds " "the exact active generation, source snapshot, recoverable index backup, and stopped-daemon " - "offline operator-maintenance ownership; it never performs a broad index reset." + "offline operator-maintenance ownership; it never performs a broad index reset. Fresh applies run through " + "OperationExecutor; only a durable already-authorized intent may resume receipt finalization offline." ), consumes=("raw_revision_heads", "raw_revision_applications", "raw_sessions"), produces=("raw_authority_index_seed_recovery_receipt",), diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 06b92baffe..b5feb24487 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -15,6 +15,8 @@ RecoveryOperation, apply_raw_authority_recovery, inspect_raw_authority_recovery, + resume_raw_authority_recovery, + write_recovery_plan, ) from polylogue.maintenance.raw_authority_reset import ( prune_orphaned_index_revision_seeds, @@ -170,6 +172,8 @@ def test_census_reset_preserves_recovery_evidence_when_final_receipt_write_fails _seed_raw(tmp_path / "source.db", "r-keep") backup = _backup_authority(tmp_path, monkeypatch, tier="source") plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + plan_file = tmp_path / "external-recovery-plan.json" + write_recovery_plan(plan, plan_file) from polylogue.maintenance import raw_authority_recovery @@ -192,10 +196,18 @@ def fail_final_receipt(path: Path, payload: dict[str, object], *, digest_field: assert not receipt_path.exists() monkeypatch.setattr(raw_authority_recovery, "_write_immutable", original_write) - recovered = apply_raw_authority_recovery(plan) + operation_id = plan.operation_id + plan_file.unlink() + recovered = resume_raw_authority_recovery( + tmp_path, + RecoveryOperation.RESET_CENSUS, + operation_id=operation_id, + ) assert recovered.status == "already_satisfied" assert recovered.receipt_path == receipt_path - assert receipt_path.is_file() + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["operation_id"] == operation_id + assert receipt["plan_digest"] == recovered.plan.plan_digest def test_recovery_receipt_path_must_be_owned_by_the_archive(tmp_path: Path) -> None: @@ -211,6 +223,24 @@ def test_recovery_receipt_path_must_be_owned_by_the_archive(tmp_path: Path) -> N ) +def test_recovery_receipt_path_rejects_archive_symlink_escape(tmp_path: Path) -> None: + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + escaped = tmp_path.parent / "escaped-recovery-receipts" + escaped.mkdir() + receipt_dir = tmp_path / ".maintenance-state" / "raw-authority-recovery" + receipt_dir.mkdir(parents=True) + (receipt_dir / "escape").symlink_to(escaped, target_is_directory=True) + + with pytest.raises(RawAuthorityRecoveryError, match="must not traverse an archive symlink"): + inspect_raw_authority_recovery( + tmp_path, + RecoveryOperation.RESET_CENSUS, + receipt_path=receipt_dir / "escape" / "receipt.json", + ) + + def test_census_reset_dry_run_does_not_mutate_and_apply_requires_backup(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _seed_ledger(tmp_path / "source.db") diff --git a/tests/unit/operations/test_mutation_census.py b/tests/unit/operations/test_mutation_census.py index 7e90906b53..02ed011055 100644 --- a/tests/unit/operations/test_mutation_census.py +++ b/tests/unit/operations/test_mutation_census.py @@ -85,3 +85,5 @@ def test_raw_authority_recovery_routes_are_executor_routed() -> None: ): assert rows[operation]["status"] == "executor-routed" assert rows[operation]["actuator"] == actuator + assert rows[operation]["execution_owner"] == "offline-operator-maintenance" + assert rows[operation]["recovery_continuation"] == "offline-durable-intent" From 5d4b952c8e70d0b5d454efc9a65e645050bf5aa4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 03:09:46 +0200 Subject: [PATCH 14/27] fix(raw-authority): harden durable receipt publication Problem: an operation-id resume could not find a permitted custom receipt path, and pathname-based publication still had a symlink replacement window.\n\nWhat changed: persist and read recovery artifacts through descriptor-relative no-follow file operations, accept the original archive-owned receipt path during resume, and prove a custom-path recovery survives a failed final receipt write.\n\nCompatibility: external plan artifacts keep their existing path behavior; durable receipt and intent artifacts remain restricted to archive-owned maintenance state. --- docs/maintenance.md | 2 +- .../maintenance/_raw_authority_recovery.py | 1 + .../maintenance/raw_authority_recovery.py | 141 ++++++++++++++++-- .../maintenance/test_raw_authority_reset.py | 19 ++- 4 files changed, 143 insertions(+), 20 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index 7bcc1bc3b7..adf83e192e 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -826,7 +826,7 @@ polylogue ops maintenance raw-authority-recovery \ --output-format json ``` -Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent, including the complete plan, under `/.maintenance-state/raw-authority-recovery/`; a restart finalizes that intent into the self-hashed receipt if the mutation committed. Receipt destinations are restricted to that archive-owned durable location and reject symlink traversal. If the external plan file was lost after a final-receipt failure, rerun `--apply --operation-id ` to resume the archive-owned intent. It does not invoke the broad index reset or reparse path. +Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent, including the complete plan, under `/.maintenance-state/raw-authority-recovery/`; a restart finalizes that intent into the self-hashed receipt if the mutation committed. Receipt destinations are restricted to that archive-owned durable location and are published through descriptor-relative no-follow operations. If the external plan file was lost after a final-receipt failure, rerun `--apply --operation-id ` to resume the archive-owned intent, retaining `--receipt-file` when the original plan used a custom archive-owned receipt destination. It does not invoke the broad index reset or reparse path. ### Measuring Codex UUID-title coverage diff --git a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py index 526e68b9c0..22e89ff139 100644 --- a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py +++ b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py @@ -70,6 +70,7 @@ def raw_authority_recovery_command( env.config.archive_root, selected, operation_id=operation_id, + receipt_path=receipt_file, ) else: raise click.ClickException("--apply requires --plan-file or --operation-id for an interrupted intent") diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index 051aa1d3e8..c7b5d155c4 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -15,8 +15,8 @@ import sqlite3 import tempfile import uuid -from collections.abc import Mapping -from contextlib import closing +from collections.abc import Generator, Mapping +from contextlib import closing, contextmanager, suppress from dataclasses import dataclass from enum import StrEnum from pathlib import Path @@ -492,6 +492,68 @@ def _read_json(path: Path) -> dict[str, object]: return {str(key): value for key, value in payload.items()} +@contextmanager +def _durable_receipt_directory( + root: Path, receipt_path: Path, *, create: bool +) -> Generator[tuple[int, str], None, None]: + """Open a receipt parent by descriptor without following archive symlinks.""" + + candidate = _resolve_receipt_path(root, receipt_path) + archive_root = root.expanduser().resolve(strict=False) + current_fd = os.open(archive_root, os.O_RDONLY | os.O_DIRECTORY) + try: + for component in candidate.parent.relative_to(archive_root).parts: + try: + next_fd = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=current_fd) + except FileNotFoundError: + if not create: + raise + with suppress(FileExistsError): + os.mkdir(component, mode=0o700, dir_fd=current_fd) + next_fd = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=current_fd) + os.close(current_fd) + current_fd = next_fd + yield current_fd, candidate.name + except FileNotFoundError: + raise + except OSError as exc: + raise RawAuthorityRecoveryError( + f"recovery receipt path must remain in the archive-owned durable location: {candidate}" + ) from exc + finally: + os.close(current_fd) + + +def _read_json_at(directory_fd: int, name: str, *, display_path: Path) -> dict[str, object]: + try: + artifact_fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd) + except FileNotFoundError: + raise + except OSError as exc: + raise RawAuthorityRecoveryError( + f"recovery artifact is not a regular archive-owned file: {display_path}" + ) from exc + try: + with os.fdopen(artifact_fd, "r", encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, ValueError) as exc: + raise RawAuthorityRecoveryError(f"recovery plan or receipt is unreadable: {display_path}") from exc + if not isinstance(payload, dict): + raise RawAuthorityRecoveryError(f"recovery artifact is not a JSON object: {display_path}") + return {str(key): value for key, value in payload.items()} + + +def _read_durable_json(root: Path, path: Path) -> dict[str, object] | None: + try: + with _durable_receipt_directory(root, path, create=False) as (directory_fd, name): + try: + return _read_json_at(directory_fd, name, display_path=path) + except FileNotFoundError: + return None + except FileNotFoundError: + return None + + def _int_mapping(payload: object, *, field: str) -> dict[str, int]: if not isinstance(payload, dict) or any(not isinstance(value, int) for value in payload.values()): raise RawAuthorityRecoveryError(f"raw-authority recovery plan field {field!r} is malformed") @@ -531,6 +593,55 @@ def _write_immutable(path: Path, payload: dict[str, object], *, digest_field: st return path +def _write_durable_immutable(root: Path, path: Path, payload: dict[str, object], *, digest_field: str) -> Path: + """Publish a self-hashed receipt without pathname traversal after validation.""" + + body = {key: value for key, value in payload.items() if key != digest_field} + stamped = {**body, digest_field: _digest(body)} + serialized = (json.dumps(stamped, indent=2, sort_keys=True) + "\n").encode("utf-8") + with _durable_receipt_directory(root, path, create=True) as (directory_fd, name): + try: + existing = _read_json_at(directory_fd, name, display_path=path) + except FileNotFoundError: + existing = None + if existing is not None: + if existing != stamped: + raise RawAuthorityRecoveryError( + f"immutable recovery artifact already exists with different content: {path}" + ) + return path + + temporary_name = f".{name}.{uuid.uuid4().hex}.tmp" + temporary_fd = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=directory_fd, + ) + try: + view = memoryview(serialized) + while view: + view = view[os.write(temporary_fd, view) :] + os.fsync(temporary_fd) + os.link( + temporary_name, + name, + src_dir_fd=directory_fd, + dst_dir_fd=directory_fd, + follow_symlinks=False, + ) + os.fsync(directory_fd) + except FileExistsError: + existing = _read_json_at(directory_fd, name, display_path=path) + if existing != stamped: + raise RawAuthorityRecoveryError(f"immutable recovery artifact race changed content: {path}") from None + finally: + os.close(temporary_fd) + with suppress(FileNotFoundError): + os.unlink(temporary_name, dir_fd=directory_fd) + return path + + def write_recovery_plan(plan: RawAuthorityRecoveryPlan, path: Path) -> Path: return _write_immutable(path, plan.to_dict(), digest_field="plan_digest") @@ -689,7 +800,8 @@ def _recovery_intent(plan: RawAuthorityRecoveryPlan) -> dict[str, object]: def _write_recovery_intent(plan: RawAuthorityRecoveryPlan) -> Path: - return _write_immutable( + return _write_durable_immutable( + Path(plan.archive_root), _intent_path(Path(plan.receipt_path)), _recovery_intent(plan), digest_field="intent_sha256", @@ -698,9 +810,9 @@ def _write_recovery_intent(plan: RawAuthorityRecoveryPlan) -> Path: def _intent_for_plan(plan: RawAuthorityRecoveryPlan) -> dict[str, object] | None: path = _intent_path(Path(plan.receipt_path)) - if not path.is_file(): + payload = _read_durable_json(Path(plan.archive_root), path) + if payload is None: return None - payload = _read_json(path) expected = payload.get("intent_sha256") if not isinstance(expected, str) or expected != _digest( {key: value for key, value in payload.items() if key != "intent_sha256"} @@ -752,7 +864,8 @@ def _write_recovery_receipt( after_counts: dict[str, int], postflight: dict[str, object], ) -> Path: - return _write_immutable( + return _write_durable_immutable( + Path(plan.archive_root), Path(plan.receipt_path), _receipt_payload(plan, before_counts=before_counts, after_counts=after_counts, postflight=postflight), digest_field="receipt_sha256", @@ -969,9 +1082,9 @@ def __init__(self) -> None: def _receipt_for_plan(plan: RawAuthorityRecoveryPlan) -> dict[str, object] | None: path = Path(plan.receipt_path) - if not path.is_file(): + payload = _read_durable_json(Path(plan.archive_root), path) + if payload is None: return None - payload = _read_json(path) expected = payload.get("receipt_sha256") if not isinstance(expected, str) or expected != _digest( {key: value for key, value in payload.items() if key != "receipt_sha256"} @@ -1003,13 +1116,14 @@ def _plan_from_intent( operation: RecoveryOperation, *, operation_id: str, + receipt_path: Path | None, ) -> RawAuthorityRecoveryPlan: root = archive_root.expanduser().resolve(strict=False) - receipt_path = _resolve_receipt_path(root, _default_receipt_path(root, operation_id)) - intent_path = _intent_path(receipt_path) - if not intent_path.is_file(): + selected_receipt_path = _resolve_receipt_path(root, receipt_path or _default_receipt_path(root, operation_id)) + intent_path = _intent_path(selected_receipt_path) + payload = _read_durable_json(root, intent_path) + if payload is None: raise RawAuthorityRecoveryError(f"no restartable recovery intent exists for operation {operation_id!r}") - payload = _read_json(intent_path) serialized_plan = payload.get("plan") if not isinstance(serialized_plan, dict): raise RawAuthorityRecoveryError("existing recovery intent does not contain a complete recovery plan") @@ -1026,11 +1140,12 @@ def resume_raw_authority_recovery( operation: RecoveryOperation | str, *, operation_id: str, + receipt_path: Path | None = None, ) -> RawAuthorityRecoveryReport: """Resume a durable intent when the external dry-run plan artifact is unavailable.""" selected = RecoveryOperation(operation) - plan = _plan_from_intent(archive_root, selected, operation_id=operation_id) + plan = _plan_from_intent(archive_root, selected, operation_id=operation_id, receipt_path=receipt_path) return apply_raw_authority_recovery(plan) diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index b5feb24487..709a5d9ac9 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -171,20 +171,26 @@ def test_census_reset_preserves_recovery_evidence_when_final_receipt_write_fails _seed_ledger(tmp_path / "source.db") _seed_raw(tmp_path / "source.db", "r-keep") backup = _backup_authority(tmp_path, monkeypatch, tier="source") - plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + receipt_path = tmp_path / ".maintenance-state" / "raw-authority-recovery" / "nested" / "recovery.json" + plan = inspect_raw_authority_recovery( + tmp_path, + RecoveryOperation.RESET_CENSUS, + backup_manifest=backup, + receipt_path=receipt_path, + ) plan_file = tmp_path / "external-recovery-plan.json" write_recovery_plan(plan, plan_file) from polylogue.maintenance import raw_authority_recovery - original_write = raw_authority_recovery._write_immutable + original_write = raw_authority_recovery._write_durable_immutable - def fail_final_receipt(path: Path, payload: dict[str, object], *, digest_field: str) -> Path: + def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, digest_field: str) -> Path: if digest_field == "receipt_sha256": raise OSError("injected final receipt write failure") - return original_write(path, payload, digest_field=digest_field) + return original_write(root, path, payload, digest_field=digest_field) - monkeypatch.setattr(raw_authority_recovery, "_write_immutable", fail_final_receipt) + monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", fail_final_receipt) with pytest.raises(RawAuthorityRecoveryError, match="injected final receipt write failure"): apply_raw_authority_recovery(plan) @@ -195,13 +201,14 @@ def fail_final_receipt(path: Path, payload: dict[str, object], *, digest_field: assert intent_path.is_file() assert not receipt_path.exists() - monkeypatch.setattr(raw_authority_recovery, "_write_immutable", original_write) + monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", original_write) operation_id = plan.operation_id plan_file.unlink() recovered = resume_raw_authority_recovery( tmp_path, RecoveryOperation.RESET_CENSUS, operation_id=operation_id, + receipt_path=receipt_path, ) assert recovered.status == "already_satisfied" assert recovered.receipt_path == receipt_path From dd117431c814248dbdb4325f5265a778fba9484b Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 03:56:29 +0200 Subject: [PATCH 15/27] fix(raw-authority): bind recovery intents to exact state Problem: count-only recovery finalization could accept altered retained index rows; receipt paths also accepted non-regular artifacts, and an uncommitted intent skipped the executor authorization boundary. What changed: persist and verify the exact retained index-seed digest, fsync each new receipt directory, reject non-regular receipt artifacts before reading, and reauthorize uncommitted intents through the maintenance executor. Compatibility: persisted index-prune intents without a post-target digest now fail closed rather than resuming against unverifiable state. --- docs/maintenance.md | 2 +- .../maintenance/raw_authority_recovery.py | 73 +++++++++-- .../maintenance/test_raw_authority_reset.py | 113 ++++++++++++++++++ 3 files changed, 180 insertions(+), 8 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index adf83e192e..5f8b201e86 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -826,7 +826,7 @@ polylogue ops maintenance raw-authority-recovery \ --output-format json ``` -Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent, including the complete plan, under `/.maintenance-state/raw-authority-recovery/`; a restart finalizes that intent into the self-hashed receipt if the mutation committed. Receipt destinations are restricted to that archive-owned durable location and are published through descriptor-relative no-follow operations. If the external plan file was lost after a final-receipt failure, rerun `--apply --operation-id ` to resume the archive-owned intent, retaining `--receipt-file` when the original plan used a custom archive-owned receipt destination. It does not invoke the broad index reset or reparse path. +Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent, including the complete plan, under `/.maintenance-state/raw-authority-recovery/`; each newly created receipt directory is fsynced before use. A restart finalizes that intent into the self-hashed receipt only when the exact committed postflight matches. An uncommitted intent goes back through PREPARE, AUTHORIZE, and EXECUTE. Receipt destinations are restricted to that archive-owned durable location and are published through descriptor-relative no-follow operations that accept regular files only. If the external plan file was lost after a final-receipt failure, rerun `--apply --operation-id ` to resume the archive-owned intent, retaining `--receipt-file` when the original plan used a custom archive-owned receipt destination. It does not invoke the broad index reset or reparse path. ### Measuring Codex UUID-title coverage diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index c7b5d155c4..8587f2a705 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -13,6 +13,7 @@ import json import os import sqlite3 +import stat as stat_module import tempfile import uuid from collections.abc import Generator, Mapping @@ -271,6 +272,32 @@ def _index_candidates(conn: sqlite3.Connection) -> dict[str, tuple[str, ...]]: return {"raw_revision_heads": heads, "raw_revision_applications": applications} +def _index_seed_digest(conn: sqlite3.Connection, *, excluded_keys: Mapping[str, tuple[str, ...]] | None = None) -> str: + """Hash the exact index-seed rows expected after a guarded prune.""" + + key_columns = { + "raw_revision_heads": "logical_source_key", + "raw_revision_applications": "decision_id", + } + payload: dict[str, object] = {} + for table, key_column in key_columns.items(): + quoted = _quote_identifier(table) + columns = [str(row[1]) for row in conn.execute(f"PRAGMA table_info({quoted})")] + if not columns: + raise RawAuthorityRecoveryError(f"cannot fingerprint missing or malformed table: {table}") + excluded = () if excluded_keys is None else excluded_keys.get(table, ()) + if excluded: + placeholders = ", ".join("?" for _ in excluded) + rows_query = f"SELECT * FROM {quoted} WHERE {_quote_identifier(key_column)} NOT IN ({placeholders})" + rows = conn.execute(rows_query, excluded) + else: + rows = conn.execute(f"SELECT * FROM {quoted}") + values = [[_value_for_digest(value) for value in row] for row in rows] + values.sort(key=_canonical_bytes) + payload[table] = {"columns": columns, "rows": values} + return _digest(payload) + + def _validate_backup(path: Path | None, *, tier: ArchiveTier, connection: sqlite3.Connection) -> dict[str, object]: if path is None: raise RawAuthorityRecoveryError(f"{tier.value}-tier backup authority is required for apply") @@ -341,6 +368,7 @@ class RawAuthorityRecoveryPlan: active_generation: dict[str, object] before_counts: dict[str, int] candidate_keys: dict[str, tuple[str, ...]] + post_target_digest: str | None protected_digest: str backup_authority: dict[str, object] | None receipt_path: str @@ -362,6 +390,7 @@ def _payload(self) -> dict[str, object]: "active_generation": self.active_generation, "before_counts": self.before_counts, "candidate_keys": {key: list(value) for key, value in self.candidate_keys.items()}, + "post_target_digest": self.post_target_digest, "protected_digest": self.protected_digest, "backup_authority": self.backup_authority, "receipt_path": self.receipt_path, @@ -400,11 +429,15 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: "source_snapshot", "active_generation", "before_counts", + "post_target_digest", "protected_digest", "receipt_path", ) if any(key not in payload for key in required): raise RawAuthorityRecoveryError("raw-authority recovery plan is missing a required field") + post_target_digest = payload["post_target_digest"] + if post_target_digest is not None and not isinstance(post_target_digest, str): + raise RawAuthorityRecoveryError("raw-authority recovery plan post-target digest is malformed") return cls( operation_id=str(payload["operation_id"]), operation=str(payload["operation"]), @@ -419,6 +452,7 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=_int_mapping(payload["before_counts"], field="before_counts"), candidate_keys=candidates, + post_target_digest=post_target_digest, protected_digest=str(payload["protected_digest"]), backup_authority=( cast(dict[str, object], payload["backup_authority"]) @@ -508,9 +542,16 @@ def _durable_receipt_directory( except FileNotFoundError: if not create: raise - with suppress(FileExistsError): + created = False + try: os.mkdir(component, mode=0o700, dir_fd=current_fd) + created = True + except FileExistsError: + pass next_fd = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=current_fd) + if created: + os.fsync(current_fd) + os.fsync(next_fd) os.close(current_fd) current_fd = next_fd yield current_fd, candidate.name @@ -526,7 +567,7 @@ def _durable_receipt_directory( def _read_json_at(directory_fd: int, name: str, *, display_path: Path) -> dict[str, object]: try: - artifact_fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd) + artifact_fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory_fd) except FileNotFoundError: raise except OSError as exc: @@ -534,10 +575,16 @@ def _read_json_at(directory_fd: int, name: str, *, display_path: Path) -> dict[s f"recovery artifact is not a regular archive-owned file: {display_path}" ) from exc try: + if not stat_module.S_ISREG(os.fstat(artifact_fd).st_mode): + raise RawAuthorityRecoveryError(f"recovery artifact is not a regular archive-owned file: {display_path}") with os.fdopen(artifact_fd, "r", encoding="utf-8") as handle: + artifact_fd = -1 payload = json.load(handle) except (OSError, ValueError) as exc: raise RawAuthorityRecoveryError(f"recovery plan or receipt is unreadable: {display_path}") from exc + finally: + if artifact_fd >= 0: + os.close(artifact_fd) if not isinstance(payload, dict): raise RawAuthorityRecoveryError(f"recovery artifact is not a JSON object: {display_path}") return {str(key): value for key, value in payload.items()} @@ -678,6 +725,7 @@ def _build_plan( _validate_integrity(source, tier="source") counts = source_counts candidate_keys: dict[str, tuple[str, ...]] = {} + post_target_digest: str | None = None protected = _protected_digest(source, excluded=_RESET_TABLES) else: with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as index: @@ -686,6 +734,7 @@ def _build_plan( _validate_integrity(index, tier="active index") candidate_keys = _index_candidates(index) counts = _count_tables(index, _INDEX_TARGETS) + post_target_digest = _index_seed_digest(index, excluded_keys=candidate_keys) protected = _protected_digest(index, excluded=_INDEX_TARGETS) source_snapshot = source_revision_snapshot(root) backup_authority: dict[str, object] | None = None @@ -711,6 +760,7 @@ def _build_plan( "active_generation": _generation_identity(root, location), "before_counts": counts, "candidate_keys": {key: list(value) for key, value in candidate_keys.items()}, + "post_target_digest": post_target_digest, "protected_digest": protected, "backup_authority": backup_authority, "receipt_path": str(receipt), @@ -729,6 +779,7 @@ def _build_plan( active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=counts, candidate_keys=candidate_keys, + post_target_digest=post_target_digest, protected_digest=protected, backup_authority=backup_authority, receipt_path=str(receipt), @@ -891,6 +942,11 @@ def _committed_postflight(plan: RawAuthorityRecoveryPlan) -> tuple[dict[str, int expected_after = {key: plan.before_counts[key] - len(plan.candidate_keys[key]) for key in _INDEX_TARGETS} if after_counts != expected_after: return None + if operation is RecoveryOperation.PRUNE_INDEX_SEEDS: + if plan.post_target_digest is None: + raise RawAuthorityRecoveryError("recovery plan is missing its exact index seed post-target digest") + if _index_seed_digest(conn) != plan.post_target_digest: + raise RawAuthorityRecoveryError("recovery intent does not match the exact committed index seed state") postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) return after_counts, postflight @@ -979,6 +1035,8 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: expected_after = {key: before_counts[key] - len(plan.candidate_keys[key]) for key in _INDEX_TARGETS} if after_counts != expected_after: raise RawAuthorityRecoveryError("index-seed prune postflight changed an unexpected row class") + if plan.post_target_digest is None or _index_seed_digest(conn) != plan.post_target_digest: + raise RawAuthorityRecoveryError("index-seed prune postflight does not match the exact target state") postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) conn.commit() except Exception: @@ -1194,10 +1252,10 @@ def apply_raw_authority_recovery( executor = OperationExecutor() try: location = ArchiveLocation.resolve(root) - # A final receipt may be missing after a process crash or I/O failure, - # but the immutable intent was fsynced before the target transaction. - # Resume that exact intent under the same offline ownership boundary; - # a new PREPARE would correctly look stale after a committed mutation. + # A final receipt may be missing after a process crash or I/O failure. + # Only exact committed postflight evidence can skip a fresh executor + # authorization. An uncommitted intent is evidence of interruption, + # not authority to perform the destructive mutation. if _intent_for_plan(selected) is not None: with OwnedArchiveLocation.acquire( location, owner_id=f"raw-authority-recovery:{selected.operation_id}" @@ -1205,7 +1263,8 @@ def apply_raw_authority_recovery( current_location = ArchiveLocation.resolve(root) assert_owns_archive_location(owned, current_location) with RebuildLease(root): - return _apply_plan(selected) + if _committed_postflight(selected) is not None: + return _apply_plan(selected) prepared = executor.prepare(actuator, args) if prepared.context.get("recovery_plan_digest") != selected.plan_digest: raise PlanStaleError("recovery plan is stale before lease acquisition") diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 709a5d9ac9..15085471f6 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -4,15 +4,18 @@ import hashlib import json +import os import shutil import sqlite3 from pathlib import Path +from typing import Never import pytest from polylogue.maintenance.raw_authority_recovery import ( RawAuthorityRecoveryError, RecoveryOperation, + _write_recovery_intent, apply_raw_authority_recovery, inspect_raw_authority_recovery, resume_raw_authority_recovery, @@ -22,6 +25,7 @@ prune_orphaned_index_revision_seeds, reset_raw_authority_census, ) +from polylogue.operations.mutation_transaction import OperationExecutor from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root @@ -248,6 +252,82 @@ def test_recovery_receipt_path_rejects_archive_symlink_escape(tmp_path: Path) -> ) +def test_recovery_fsyncs_each_new_durable_receipt_directory(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The production receipt writer durably records every created directory entry.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + receipt_path = tmp_path / ".maintenance-state" / "raw-authority-recovery" / "nested" / "recovery.json" + plan = inspect_raw_authority_recovery( + tmp_path, + RecoveryOperation.RESET_CENSUS, + backup_manifest=backup, + receipt_path=receipt_path, + ) + + original_fsync = os.fsync + synced_inodes: set[int] = set() + + def record_fsync(fd: int) -> None: + synced_inodes.add(os.fstat(fd).st_ino) + original_fsync(fd) + + monkeypatch.setattr(os, "fsync", record_fsync) + assert apply_raw_authority_recovery(plan).status == "applied" + + expected = { + (tmp_path / ".maintenance-state").stat().st_ino, + (tmp_path / ".maintenance-state" / "raw-authority-recovery").stat().st_ino, + receipt_path.parent.stat().st_ino, + } + assert expected <= synced_inodes + + +def test_recovery_receipt_rejects_fifo_before_reading(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A non-regular artifact must not be opened as a receipt stream.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + receipt_path = Path(plan.receipt_path) + receipt_path.parent.mkdir(parents=True) + os.mkfifo(receipt_path) + holder_fd = os.open(receipt_path, os.O_RDWR | os.O_NONBLOCK) + os.write(holder_fd, b"{}") + try: + with pytest.raises(RawAuthorityRecoveryError, match="regular archive-owned file"): + apply_raw_authority_recovery(plan) + finally: + os.close(holder_fd) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + + +def test_uncommitted_recovery_intent_reauthorizes_through_executor( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An intent without committed postflight state is not authorization to mutate.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + + _write_recovery_intent(plan) + + def require_authorization(*_args: object, **_kwargs: object) -> Never: + raise RuntimeError("executor authorization was required") + + monkeypatch.setattr(OperationExecutor, "authorize", require_authorization) + with pytest.raises(RuntimeError, match="executor authorization was required"): + apply_raw_authority_recovery(plan) + + def test_census_reset_dry_run_does_not_mutate_and_apply_requires_backup(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _seed_ledger(tmp_path / "source.db") @@ -374,6 +454,39 @@ def test_index_prune_refuses_stale_active_pointer_and_wrong_backup( apply_raw_authority_recovery(plan, backup_manifest=tmp_path / "different.json") +def test_index_prune_resume_refuses_changed_retained_seed_rows(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Receipt finalization requires the exact retained target rows from the committed prune.""" + + initialize_active_archive_root(tmp_path) + active_index = _seed_index_seeds(tmp_path) + backup = _backup_authority(tmp_path, monkeypatch, tier="index") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) + + from polylogue.maintenance import raw_authority_recovery + + original_write = raw_authority_recovery._write_durable_immutable + + def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, digest_field: str) -> Path: + if digest_field == "receipt_sha256": + raise OSError("injected final receipt write failure") + return original_write(root, path, payload, digest_field=digest_field) + + monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", fail_final_receipt) + with pytest.raises(RawAuthorityRecoveryError, match="injected final receipt write failure"): + apply_raw_authority_recovery(plan) + monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", original_write) + + with sqlite3.connect(active_index) as conn: + conn.execute("UPDATE raw_revision_heads SET session_id = 'changed' WHERE logical_source_key = 'k-r-present'") + + with pytest.raises(RawAuthorityRecoveryError, match="exact committed index seed state"): + resume_raw_authority_recovery( + tmp_path, + RecoveryOperation.PRUNE_INDEX_SEEDS, + operation_id=plan.operation_id, + ) + + def test_named_compatibility_facade_requires_index_backup(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _seed_index_seeds(tmp_path) From f9e593eb86fe9dacdd5de719eaccd4f6b1214ba1 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 05:26:42 +0200 Subject: [PATCH 16/27] fix(raw-authority): preserve recovery continuity Persist recovery artifacts with stable archive identity, bind the CLI operation to its plan, and extend source reset through the durable source-train continuity protocol.\n\nRecord exact retained index candidate state for restart-safe pruning while allowing append-only successors, avoid unbounded SQLite bindings, fsync all receipt parents, and report correct actuator targets. Older index intents without retained state fail closed. --- docs/maintenance.md | 4 +- polylogue/artifacts/runtime.py | 2 +- .../maintenance/_raw_authority_recovery.py | 7 +- .../maintenance/raw_authority_recovery.py | 191 +++++++++++++----- polylogue/operations/mutation_transaction.py | 2 +- .../storage/sqlite/durable_change_train.py | 83 +++++++- .../unit/cli/test_archive_maintenance_cli.py | 50 ++++- .../maintenance/test_raw_authority_reset.py | 182 +++++++++++++++++ 8 files changed, 461 insertions(+), 60 deletions(-) diff --git a/docs/maintenance.md b/docs/maintenance.md index 5f8b201e86..88e1f4c7fe 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -806,7 +806,7 @@ the typed application receipt. `polylogue ops maintenance raw-authority-frontier records an inspection census only; it has no manual plan selector or apply option. -### `polylogue ops maintenance raw-authority-recovery` - break-glass ledger recovery +### `polylogue ops maintenance raw-authority-recovery` - guarded offline ledger recovery This command family is the only operator route for the two callerless raw-authority recovery actuators. It is inspect-only by default. The census reset removes only the five poisoned census-planning tables after a verified source-tier backup. The index-seed prune removes only active-index `raw_revision_heads` and `raw_revision_applications` rows whose source raw is absent. Parser census rows, source raws, blob receipts, and present-source revision rows are outside both target sets. @@ -826,7 +826,7 @@ polylogue ops maintenance raw-authority-recovery \ --output-format json ``` -Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent, including the complete plan, under `/.maintenance-state/raw-authority-recovery/`; each newly created receipt directory is fsynced before use. A restart finalizes that intent into the self-hashed receipt only when the exact committed postflight matches. An uncommitted intent goes back through PREPARE, AUTHORIZE, and EXECUTE. Receipt destinations are restricted to that archive-owned durable location and are published through descriptor-relative no-follow operations that accept regular files only. If the external plan file was lost after a final-receipt failure, rerun `--apply --operation-id ` to resume the archive-owned intent, retaining `--receipt-file` when the original plan used a custom archive-owned receipt destination. It does not invoke the broad index reset or reparse path. +Apply is explicitly an offline operator-maintenance route, not a daemon-writer route. It refuses a running daemon, stale plan or active pointer, changed tier bytes or schema versions, malformed ledger, unexpected candidate set, mismatched backup authority, or changed unrelated rows. It acquires archive ownership and the rebuild lease before revalidation. Before the SQLite mutation it persists an fsynced immutable intent, including the complete plan, under `/.maintenance-state/raw-authority-recovery/`; a source-ledger reset also persists the established source-train continuity intent before committing. Each receipt-directory parent is fsynced while walked. A restart finalizes that intent into the self-hashed receipt only when the planned candidate rows are absent and every planned retained row matches; later append-only index successors are allowed. An uncommitted intent goes back through PREPARE, AUTHORIZE, and EXECUTE. Receipt destinations are restricted to that archive-owned durable location and are published through descriptor-relative no-follow operations that accept regular files only. If the external plan file was lost after a final-receipt failure, rerun `--apply --operation-id ` to resume the archive-owned intent, retaining `--receipt-file` when the original plan used a custom archive-owned receipt destination. It does not invoke the broad index reset or reparse path. ### Measuring Codex UUID-title coverage diff --git a/polylogue/artifacts/runtime.py b/polylogue/artifacts/runtime.py index a385270d83..5614683017 100644 --- a/polylogue/artifacts/runtime.py +++ b/polylogue/artifacts/runtime.py @@ -669,7 +669,7 @@ ), ArtifactNode( name="raw_authority_index_seed_recovery_receipt", - layer=ArtifactLayer.PROJECTION, + layer=ArtifactLayer.DURABLE, description="Immutable receipt for a guarded active-index orphan-seed prune.", depends_on=("raw_revision_heads", "raw_revision_applications"), code_refs=("polylogue.maintenance.raw_authority_recovery",), diff --git a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py index 22e89ff139..80bd55458d 100644 --- a/polylogue/cli/commands/maintenance/_raw_authority_recovery.py +++ b/polylogue/cli/commands/maintenance/_raw_authority_recovery.py @@ -64,7 +64,12 @@ def raw_authority_recovery_command( selected = RecoveryOperation(operation) if apply_changes: if plan_file is not None: - report = apply_raw_authority_recovery(plan_file, backup_manifest=backup_manifest) + artifact = RawAuthorityRecoveryPlan.from_dict(json.loads(plan_file.read_text(encoding="utf-8"))) + if artifact.operation != selected.value: + raise click.ClickException( + f"plan file declares operation {artifact.operation!r}, but --operation is {selected.value!r}" + ) + report = apply_raw_authority_recovery(artifact, backup_manifest=backup_manifest) elif operation_id is not None: report = resume_raw_authority_recovery( env.config.archive_root, diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index 8587f2a705..dc3de167a7 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -1,6 +1,6 @@ """Guarded, receipted recovery for the raw-authority derived state. -This is the only production mutation route for the two break-glass repairs in +This is the only production mutation route for the two guarded repairs in this module. The storage helpers expose counts for diagnostics, but they do not authorize deletion. A plan is an exact snapshot of one archive and one operation. APPLY rechecks that snapshot while holding both archive ownership @@ -48,7 +48,13 @@ from polylogue.storage.introspection import table_exists from polylogue.storage.sqlite.archive_tiers import ARCHIVE_VERSION_BY_TIER from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier +from polylogue.storage.sqlite.durable_change_train import ( + DurableChangeTrainError, + reconcile_durable_change_train_startup, + write_source_continuity_pending_intent, +) from polylogue.storage.sqlite.migration_runner import ( + capture_durable_database_evidence, validate_backup_manifest_covers_derived_tier, validate_migration_backup_manifest, ) @@ -184,7 +190,14 @@ def _archive_identity(root: Path, location: ArchiveLocation) -> dict[str, object from polylogue.storage.archive_identity import ArchiveIdentity identity = ArchiveIdentity.resolve_location(location) - return identity.as_dict(unit="raw-authority-recovery") + # Plans are deliberately portable across CLI invocations. The complete + # identity includes useful process diagnostics, but those cannot be part + # of the durable archive identity because they change at every restart. + return { + key: value + for key, value in identity.as_dict(unit="raw-authority-recovery").items() + if key not in {"process_id", "executable", "invocation_id"} + } def _generation_identity(root: Path, location: ArchiveLocation) -> dict[str, object]: @@ -221,8 +234,8 @@ def _validate_ledger(conn: sqlite3.Connection) -> None: "raw_authority_blockers": ("expected_json", "observed_json"), }.items(): quoted = _quote_identifier(table) + names = [str(item[1]) for item in conn.execute(f"PRAGMA table_info({quoted})")] for row in conn.execute(f"SELECT * FROM {quoted}"): - names = [str(item[1]) for item in conn.execute(f"PRAGMA table_info({quoted})")] values = dict(zip(names, row, strict=True)) for column in columns: try: @@ -272,30 +285,59 @@ def _index_candidates(conn: sqlite3.Connection) -> dict[str, tuple[str, ...]]: return {"raw_revision_heads": heads, "raw_revision_applications": applications} -def _index_seed_digest(conn: sqlite3.Connection, *, excluded_keys: Mapping[str, tuple[str, ...]] | None = None) -> str: - """Hash the exact index-seed rows expected after a guarded prune.""" +def _index_seed_rows( + conn: sqlite3.Connection, *, excluded_keys: Mapping[str, tuple[str, ...]] | None = None +) -> dict[str, dict[str, str]]: + """Return stable row digests keyed by each index-seed primary key.""" key_columns = { "raw_revision_heads": "logical_source_key", "raw_revision_applications": "decision_id", } - payload: dict[str, object] = {} + payload: dict[str, dict[str, str]] = {} for table, key_column in key_columns.items(): quoted = _quote_identifier(table) columns = [str(row[1]) for row in conn.execute(f"PRAGMA table_info({quoted})")] if not columns: raise RawAuthorityRecoveryError(f"cannot fingerprint missing or malformed table: {table}") - excluded = () if excluded_keys is None else excluded_keys.get(table, ()) - if excluded: - placeholders = ", ".join("?" for _ in excluded) - rows_query = f"SELECT * FROM {quoted} WHERE {_quote_identifier(key_column)} NOT IN ({placeholders})" - rows = conn.execute(rows_query, excluded) - else: - rows = conn.execute(f"SELECT * FROM {quoted}") - values = [[_value_for_digest(value) for value in row] for row in rows] - values.sort(key=_canonical_bytes) - payload[table] = {"columns": columns, "rows": values} - return _digest(payload) + key_index = columns.index(key_column) + excluded = set(() if excluded_keys is None else excluded_keys.get(table, ())) + row_digests: dict[str, str] = {} + for row in conn.execute(f"SELECT * FROM {quoted}"): + key = str(row[key_index]) + if key in excluded: + continue + if key in row_digests: + raise RawAuthorityRecoveryError(f"index seed table has a duplicate primary key: {table}.{key_column}") + row_digests[key] = _digest({"columns": columns, "row": [_value_for_digest(value) for value in row]}) + payload[table] = row_digests + return payload + + +def _index_seed_digest(conn: sqlite3.Connection, *, excluded_keys: Mapping[str, tuple[str, ...]] | None = None) -> str: + """Hash a bounded, key-addressable index-seed row snapshot.""" + + return _digest(_index_seed_rows(conn, excluded_keys=excluded_keys)) + + +def _verify_index_seed_post_target(conn: sqlite3.Connection, plan: RawAuthorityRecoveryPlan) -> None: + """Require planned retained rows and candidate absence, allowing append-only successors.""" + + if plan.post_target_rows is None or plan.post_target_digest is None: + raise RawAuthorityRecoveryError("recovery plan is missing its exact index seed post-target state") + if _digest(plan.post_target_rows) != plan.post_target_digest: + raise RawAuthorityRecoveryError("recovery plan has an invalid exact index seed post-target digest") + observed = _index_seed_rows(conn) + for table in _INDEX_TARGETS: + candidate_keys = set(plan.candidate_keys[table]) + if candidate_keys & set(observed[table]): + raise RawAuthorityRecoveryError("recovery intent still contains a planned orphaned index seed") + expected = plan.post_target_rows.get(table) + if not isinstance(expected, dict): + raise RawAuthorityRecoveryError("recovery plan has malformed index seed post-target rows") + retained = {key: observed[table].get(key) for key in expected} + if retained != expected: + raise RawAuthorityRecoveryError("recovery intent does not match the exact committed index seed state") def _validate_backup(path: Path | None, *, tier: ArchiveTier, connection: sqlite3.Connection) -> dict[str, object]: @@ -368,6 +410,7 @@ class RawAuthorityRecoveryPlan: active_generation: dict[str, object] before_counts: dict[str, int] candidate_keys: dict[str, tuple[str, ...]] + post_target_rows: dict[str, dict[str, str]] | None post_target_digest: str | None protected_digest: str backup_authority: dict[str, object] | None @@ -390,6 +433,7 @@ def _payload(self) -> dict[str, object]: "active_generation": self.active_generation, "before_counts": self.before_counts, "candidate_keys": {key: list(value) for key, value in self.candidate_keys.items()}, + "post_target_rows": self.post_target_rows, "post_target_digest": self.post_target_digest, "protected_digest": self.protected_digest, "backup_authority": self.backup_authority, @@ -429,12 +473,23 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: "source_snapshot", "active_generation", "before_counts", + "post_target_rows", "post_target_digest", "protected_digest", "receipt_path", ) if any(key not in payload for key in required): raise RawAuthorityRecoveryError("raw-authority recovery plan is missing a required field") + post_target_rows = payload["post_target_rows"] + if post_target_rows is not None and ( + not isinstance(post_target_rows, dict) + or any( + not isinstance(rows, dict) + or any(not isinstance(key, str) or not isinstance(value, str) for key, value in rows.items()) + for rows in post_target_rows.values() + ) + ): + raise RawAuthorityRecoveryError("raw-authority recovery plan post-target rows are malformed") post_target_digest = payload["post_target_digest"] if post_target_digest is not None and not isinstance(post_target_digest, str): raise RawAuthorityRecoveryError("raw-authority recovery plan post-target digest is malformed") @@ -452,6 +507,7 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=_int_mapping(payload["before_counts"], field="before_counts"), candidate_keys=candidates, + post_target_rows=cast(dict[str, dict[str, str]] | None, post_target_rows), post_target_digest=post_target_digest, protected_digest=str(payload["protected_digest"]), backup_authority=( @@ -542,16 +598,14 @@ def _durable_receipt_directory( except FileNotFoundError: if not create: raise - created = False - try: + with suppress(FileExistsError): os.mkdir(component, mode=0o700, dir_fd=current_fd) - created = True - except FileExistsError: - pass next_fd = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=current_fd) - if created: - os.fsync(current_fd) - os.fsync(next_fd) + # A previous process may have created the directory entry but died + # before syncing it. Persist every surviving parent while walking + # the descriptor chain, not only entries made by this invocation. + os.fsync(current_fd) + os.fsync(next_fd) os.close(current_fd) current_fd = next_fd yield current_fd, candidate.name @@ -725,6 +779,7 @@ def _build_plan( _validate_integrity(source, tier="source") counts = source_counts candidate_keys: dict[str, tuple[str, ...]] = {} + post_target_rows: dict[str, dict[str, str]] | None = None post_target_digest: str | None = None protected = _protected_digest(source, excluded=_RESET_TABLES) else: @@ -734,7 +789,8 @@ def _build_plan( _validate_integrity(index, tier="active index") candidate_keys = _index_candidates(index) counts = _count_tables(index, _INDEX_TARGETS) - post_target_digest = _index_seed_digest(index, excluded_keys=candidate_keys) + post_target_rows = _index_seed_rows(index, excluded_keys=candidate_keys) + post_target_digest = _digest(post_target_rows) protected = _protected_digest(index, excluded=_INDEX_TARGETS) source_snapshot = source_revision_snapshot(root) backup_authority: dict[str, object] | None = None @@ -760,6 +816,7 @@ def _build_plan( "active_generation": _generation_identity(root, location), "before_counts": counts, "candidate_keys": {key: list(value) for key, value in candidate_keys.items()}, + "post_target_rows": post_target_rows, "post_target_digest": post_target_digest, "protected_digest": protected, "backup_authority": backup_authority, @@ -779,6 +836,7 @@ def _build_plan( active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=counts, candidate_keys=candidate_keys, + post_target_rows=post_target_rows, post_target_digest=post_target_digest, protected_digest=protected, backup_authority=backup_authority, @@ -850,6 +908,39 @@ def _recovery_intent(plan: RawAuthorityRecoveryPlan) -> dict[str, object]: } +def _write_source_continuity_pending_intent(plan: RawAuthorityRecoveryPlan) -> None: + """Persist source-train refresh evidence before the reset can commit.""" + + authority = plan.backup_authority + if not isinstance(authority, dict): + raise RawAuthorityRecoveryError("source recovery plan has no backup authority for continuity refresh") + root = Path(plan.archive_root) + try: + with closing(sqlite3.connect(f"file:{root / 'source.db'}?mode=ro", uri=True)) as conn: + before = capture_durable_database_evidence(conn, ArchiveTier.SOURCE) + write_source_continuity_pending_intent( + root, + mutation_receipt=Path(plan.receipt_path), + backup_manifest=Path(str(authority["manifest_path"])), + pre_mutation_evidence=before, + operation_id=plan.operation_id, + evidence_ref=f"proof:raw-authority-recovery:{plan.plan_digest}", + ) + except (DurableChangeTrainError, KeyError, OSError, sqlite3.Error) as exc: + raise RawAuthorityRecoveryError(f"could not persist source continuity recovery intent: {exc}") from exc + + +def _refresh_source_train_continuity(plan: RawAuthorityRecoveryPlan) -> None: + """Run the established pending-intent recovery after a source receipt exists.""" + + if RecoveryOperation(plan.operation) is not RecoveryOperation.RESET_CENSUS: + return + try: + reconcile_durable_change_train_startup(Path(plan.archive_root)) + except DurableChangeTrainError as exc: + raise RawAuthorityRecoveryError(f"source continuity refresh remains incomplete: {exc}") from exc + + def _write_recovery_intent(plan: RawAuthorityRecoveryPlan) -> Path: return _write_durable_immutable( Path(plan.archive_root), @@ -940,13 +1031,10 @@ def _committed_postflight(plan: RawAuthorityRecoveryPlan) -> tuple[dict[str, int conn.execute("ATTACH DATABASE ? AS src", (str(root / "source.db"),)) after_counts = _count_tables(conn, _INDEX_TARGETS) expected_after = {key: plan.before_counts[key] - len(plan.candidate_keys[key]) for key in _INDEX_TARGETS} - if after_counts != expected_after: - return None if operation is RecoveryOperation.PRUNE_INDEX_SEEDS: - if plan.post_target_digest is None: - raise RawAuthorityRecoveryError("recovery plan is missing its exact index seed post-target digest") - if _index_seed_digest(conn) != plan.post_target_digest: - raise RawAuthorityRecoveryError("recovery intent does not match the exact committed index seed state") + _verify_index_seed_post_target(conn, plan) + elif after_counts != expected_after: + return None postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) return after_counts, postflight @@ -971,6 +1059,7 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: after_counts=after_counts, postflight=postflight, ) + _refresh_source_train_continuity(plan) return RawAuthorityRecoveryReport( plan=plan, applied=False, @@ -980,6 +1069,8 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: postflight=postflight, ) _write_recovery_intent(plan) + if operation is RecoveryOperation.RESET_CENSUS: + _write_source_continuity_pending_intent(plan) if operation is RecoveryOperation.RESET_CENSUS: with closing(sqlite3.connect(source_db)) as conn: conn.row_factory = sqlite3.Row @@ -1035,8 +1126,7 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: expected_after = {key: before_counts[key] - len(plan.candidate_keys[key]) for key in _INDEX_TARGETS} if after_counts != expected_after: raise RawAuthorityRecoveryError("index-seed prune postflight changed an unexpected row class") - if plan.post_target_digest is None or _index_seed_digest(conn) != plan.post_target_digest: - raise RawAuthorityRecoveryError("index-seed prune postflight does not match the exact target state") + _verify_index_seed_post_target(conn, plan) postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) conn.commit() except Exception: @@ -1048,6 +1138,7 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: after_counts=after_counts, postflight=postflight, ) + _refresh_source_train_continuity(plan) return RawAuthorityRecoveryReport( plan=plan, applied=True, @@ -1083,15 +1174,15 @@ def prepare(self, args: _RecoveryArgs) -> MutationPlan: backup_manifest=args.backup_manifest, receipt_path=args.receipt_path, ) + target_ref = ( + make_target_ref("source", args.operation.value) + if args.operation is RecoveryOperation.RESET_CENSUS + else make_target_ref("index", args.operation.value) + ) return build_plan( operation=self.operation, destructive_class=self.destructive_class, - target_refs=( - make_target_ref( - "source", - args.operation.value, - ), - ), + target_refs=(target_ref,), affected_tiers=("source",) if args.operation is RecoveryOperation.RESET_CENSUS else ("index",), reversible=False, context={"recovery_plan_digest": live.plan_digest, "operation_id": args.operation_id}, @@ -1109,17 +1200,22 @@ def apply(self, plan: MutationPlan, args: _RecoveryArgs) -> MutationReceipt: if live.plan_digest != args.expected_plan_digest: raise PlanStaleError("raw-authority recovery plan digest changed before apply") report = _apply_plan(live) + target_ref = ( + make_target_ref("source", args.operation.value) + if args.operation is RecoveryOperation.RESET_CENSUS + else make_target_ref("index", args.operation.value) + ) + affected_count = ( + sum(report.plan.before_counts.values()) + if args.operation is RecoveryOperation.RESET_CENSUS + else sum(len(keys) for keys in report.plan.candidate_keys.values()) + ) return MutationReceipt( operation=self.operation, plan_hash=args.expected_plan_digest, status="applied" if report.applied else "already_satisfied", - target_refs=( - make_target_ref( - "source", - args.operation.value, - ), - ), - affected_count=sum(report.plan.before_counts.values()), + target_refs=(target_ref,), + affected_count=affected_count if report.applied else 0, detail=None, receipt_ref=str(report.receipt_path) if report.receipt_path is not None else None, applied_at="recovery", @@ -1224,6 +1320,7 @@ def apply_raw_authority_recovery( if existing is not None: _require_apply_preconditions(Path(selected.archive_root)) _validate_existing_receipt(selected, existing) + _refresh_source_train_continuity(selected) return RawAuthorityRecoveryReport( plan=selected, applied=False, diff --git a/polylogue/operations/mutation_transaction.py b/polylogue/operations/mutation_transaction.py index 04782e3d15..7ea937ee4a 100644 --- a/polylogue/operations/mutation_transaction.py +++ b/polylogue/operations/mutation_transaction.py @@ -832,7 +832,7 @@ def execute( return actuator.apply(plan, args) -def make_target_ref(kind: Literal["session", "message", "block", "source"], value: object) -> str: +def make_target_ref(kind: Literal["session", "message", "block", "source", "index"], value: object) -> str: """Return a stable ``kind:value`` target ref, the shared vocabulary for plans/receipts.""" return f"{kind}:{value}" diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 64b0f60731..d3eeac11de 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -743,7 +743,7 @@ def _recover_pending_source_continuity_intents(archive_root: Path) -> None: evidence_ref = str(raw["evidence_ref"]) except (DurableChangeTrainError, KeyError, TypeError, ValueError) as exc: raise DurableChangeTrainError(f"source continuity pending intent is malformed: {path}") from exc - receipt_phase = _liveness_receipt_phase(receipt) + receipt_phase = _source_mutation_receipt_phase(receipt) if receipt_phase == "recovered_rolled_back": clear_source_continuity_pending_intent(path) continue @@ -800,6 +800,20 @@ def validate_recovered_postcondition(pending_path: Path = path) -> None: clear_source_continuity_pending_intent(path) +def _source_mutation_receipt_phase(receipt_path: Path) -> str: + """Classify a committed source-maintenance receipt without guessing its format.""" + + try: + raw = json.loads(receipt_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return _liveness_receipt_phase(receipt_path) + except (OSError, UnicodeDecodeError) as exc: + raise DurableChangeTrainError(f"source continuity pending receipt is unreadable: {receipt_path}") from exc + if isinstance(raw, dict) and raw.get("format") == "polylogue.raw-authority-recovery-receipt.v1": + return "committed" + return _liveness_receipt_phase(receipt_path) + + def _liveness_receipt_phase(receipt_path: Path) -> str: """Read the last liveness phase before deciding how a pending intent recovers.""" try: @@ -908,6 +922,71 @@ def _validate_liveness_receipt_bytes( return header +def _validate_raw_authority_reset_receipt( + payload: dict[str, object], + *, + source_path: Path, + backup_manifest: Path, + operation_id: str, +) -> dict[str, object]: + """Authenticate a self-hashed source reset receipt for continuity refresh.""" + + receipt_sha256 = payload.pop("receipt_sha256", None) + if receipt_sha256 != _canonical_json_sha256(payload): + raise DurableChangeTrainError("source mutation receipt checksum mismatch") + authority = payload.get("backup_authority") + if not isinstance(authority, dict): + raise DurableChangeTrainError("source mutation receipt has no backup authority") + backup_digest = hashlib.sha256(backup_manifest.read_bytes()).hexdigest() + if ( + payload.get("format") != "polylogue.raw-authority-recovery-receipt.v1" + or payload.get("operation") != "reset_raw_authority_census" + or payload.get("operation_id") != operation_id + or payload.get("archive_root") != str(source_path.parent) + or authority.get("tier") != ArchiveTier.SOURCE.value + or authority.get("manifest_path") != str(backup_manifest) + or authority.get("manifest_sha256") != backup_digest + ): + raise DurableChangeTrainError("source mutation receipt does not bind the named raw-authority reset") + after_counts = payload.get("after_counts") + if not isinstance(after_counts, dict) or any(value != 0 for value in after_counts.values()): + raise DurableChangeTrainError("source mutation receipt does not prove the raw-authority ledger reset") + return {"backup_manifest_sha256": backup_digest} + + +def _validate_source_mutation_receipt_bytes( + receipt_bytes: bytes, + *, + source_path: Path, + backup_manifest: Path, + operation_id: str, +) -> dict[str, object]: + """Authenticate the supported durable source-mutation receipt formats.""" + + try: + raw = json.loads(receipt_bytes) + except json.JSONDecodeError: + return _validate_liveness_receipt_bytes( + receipt_bytes, + source_path=source_path, + backup_manifest=backup_manifest, + operation_id=operation_id, + ) + if isinstance(raw, dict) and raw.get("format") == "polylogue.raw-authority-recovery-receipt.v1": + return _validate_raw_authority_reset_receipt( + cast(dict[str, object], raw), + source_path=source_path, + backup_manifest=backup_manifest, + operation_id=operation_id, + ) + return _validate_liveness_receipt_bytes( + receipt_bytes, + source_path=source_path, + backup_manifest=backup_manifest, + operation_id=operation_id, + ) + + def _validate_source_continuity_refresh_receipt( archive_root: Path, train: DurableChangeTrain, @@ -1024,7 +1103,7 @@ def _refresh_released_source_train_continuity_locked( receipt_bytes = mutation_receipt.read_bytes() except OSError as exc: raise DurableChangeTrainError("source mutation receipt is not readable") from exc - header = _validate_liveness_receipt_bytes( + header = _validate_source_mutation_receipt_bytes( receipt_bytes, source_path=source_path, backup_manifest=backup_manifest, diff --git a/tests/unit/cli/test_archive_maintenance_cli.py b/tests/unit/cli/test_archive_maintenance_cli.py index d5fb6e99cc..cfa53fca68 100644 --- a/tests/unit/cli/test_archive_maintenance_cli.py +++ b/tests/unit/cli/test_archive_maintenance_cli.py @@ -21,6 +21,11 @@ from polylogue.config import Config from polylogue.core.enums import Provider from polylogue.core.json import json_document +from polylogue.maintenance.raw_authority_recovery import ( + RecoveryOperation, + inspect_raw_authority_recovery, + write_recovery_plan, +) from polylogue.maintenance.replay import rebuild_index_from_source from polylogue.sources.revision_backfill import census_historical_revision_evidence from polylogue.storage.blob_gc import read_gc_history @@ -1910,15 +1915,9 @@ def test_cursor_authority_reconcile_cli_rejects_mixed_or_missing_mode_options( assert "requires" in result.output or "accepts only" in result.output -@pytest.mark.parametrize( - ("option", "value"), - (("--apply-plan", "raw-authority-frontier:" + "a" * 64), ("--preview-census", "census"), ("--yes", None)), -) def test_raw_authority_frontier_cli_inspects_without_applying_plans( cli_workspace: dict[str, Path], cli_runner: CliRunner, - option: str, - value: str | None, ) -> None: result = cli_runner.invoke( cli, @@ -1964,6 +1963,16 @@ def test_raw_authority_frontier_cli_inspects_without_applying_plans( for removed in ("--apply-plan", "--preview-census", "--yes"): assert removed not in frontier_help.output + +@pytest.mark.parametrize( + ("option", "value"), + (("--apply-plan", "raw-authority-frontier:" + "a" * 64), ("--preview-census", "census"), ("--yes", None)), +) +def test_raw_authority_frontier_cli_rejects_removed_apply_options( + cli_runner: CliRunner, + option: str, + value: str | None, +) -> None: rejected = cli_runner.invoke( cli, [ @@ -1979,6 +1988,35 @@ def test_raw_authority_frontier_cli_inspects_without_applying_plans( assert f"No such option {option!r}." in rejected.output +def test_raw_authority_recovery_cli_refuses_plan_for_another_operation( + cli_workspace: dict[str, Path], cli_runner: CliRunner, tmp_path: Path +) -> None: + """The required operation is bound before a destructive plan artifact is consumed.""" + + plan = inspect_raw_authority_recovery(cli_workspace["archive_root"], RecoveryOperation.RESET_CENSUS) + plan_file = tmp_path / "census-reset.plan.json" + write_recovery_plan(plan, plan_file) + + result = cli_runner.invoke( + cli, + [ + "--plain", + "ops", + "maintenance", + "raw-authority-recovery", + "--operation", + RecoveryOperation.PRUNE_INDEX_SEEDS.value, + "--apply", + "--plan-file", + str(plan_file), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 1 + assert "plan file declares operation 'reset_raw_authority_census'" in result.output + + def test_raw_authority_frontier_cli_refuses_durable_census_while_daemon_runs( cli_workspace: dict[str, Path], cli_runner: CliRunner, diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 15085471f6..a0fadc1d71 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -13,8 +13,11 @@ import pytest from polylogue.maintenance.raw_authority_recovery import ( + PruneOrphanedIndexRevisionSeedsActuator, RawAuthorityRecoveryError, RecoveryOperation, + _index_seed_digest, + _RecoveryArgs, _write_recovery_intent, apply_raw_authority_recovery, inspect_raw_authority_recovery, @@ -27,6 +30,8 @@ ) from polylogue.operations.mutation_transaction import OperationExecutor from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.durable_change_train import write_source_continuity_pending_intent +from polylogue.storage.sqlite.migration_runner import DurableDatabaseEvidence def _seed_ledger(source_db: Path) -> None: @@ -221,6 +226,68 @@ def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, di assert receipt["plan_digest"] == recovered.plan.plan_digest +def test_persisted_recovery_plan_ignores_process_scoped_archive_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A plan written by one CLI process remains authorized in the next one.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + plan_file = tmp_path / "persisted-recovery-plan.json" + write_recovery_plan(plan, plan_file) + + monkeypatch.setattr("polylogue.storage.archive_identity.os.getpid", lambda: 987654) + assert apply_raw_authority_recovery(plan_file).status == "applied" + + +def test_census_reset_persists_source_continuity_intent_before_commit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The durable source-train protocol receives pre-mutation evidence before deletion.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + + original_write_pending = write_source_continuity_pending_intent + pending_paths: list[Path] = [] + + def record_pending( + archive_root: Path, + *, + mutation_receipt: Path, + backup_manifest: Path, + pre_mutation_evidence: DurableDatabaseEvidence, + operation_id: str, + evidence_ref: str, + ) -> Path: + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + pending = original_write_pending( + archive_root, + mutation_receipt=mutation_receipt, + backup_manifest=backup_manifest, + pre_mutation_evidence=pre_mutation_evidence, + operation_id=operation_id, + evidence_ref=evidence_ref, + ) + pending_paths.append(pending) + assert pending.is_file() + return pending + + monkeypatch.setattr( + "polylogue.maintenance.raw_authority_recovery.write_source_continuity_pending_intent", record_pending + ) + assert apply_raw_authority_recovery(plan).status == "applied" + assert len(pending_paths) == 1 + assert not pending_paths[0].exists() + + def test_recovery_receipt_path_must_be_owned_by_the_archive(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _seed_ledger(tmp_path / "source.db") @@ -285,6 +352,32 @@ def record_fsync(fd: int) -> None: assert expected <= synced_inodes +def test_recovery_fsyncs_surviving_receipt_parents_after_interrupted_creation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A resumed writer persists parent entries it did not create itself.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + receipt_parent = tmp_path / ".maintenance-state" / "raw-authority-recovery" + receipt_parent.mkdir(parents=True) + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + + original_fsync = os.fsync + synced_inodes: set[int] = set() + + def record_fsync(fd: int) -> None: + synced_inodes.add(os.fstat(fd).st_ino) + original_fsync(fd) + + monkeypatch.setattr(os, "fsync", record_fsync) + assert apply_raw_authority_recovery(plan).status == "applied" + assert (tmp_path / ".maintenance-state").stat().st_ino in synced_inodes + assert receipt_parent.stat().st_ino in synced_inodes + + def test_recovery_receipt_rejects_fifo_before_reading(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A non-regular artifact must not be opened as a receipt stream.""" @@ -326,6 +419,8 @@ def require_authorization(*_args: object, **_kwargs: object) -> Never: monkeypatch.setattr(OperationExecutor, "authorize", require_authorization) with pytest.raises(RuntimeError, match="executor authorization was required"): apply_raw_authority_recovery(plan) + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) def test_census_reset_dry_run_does_not_mutate_and_apply_requires_backup(tmp_path: Path) -> None: @@ -437,6 +532,24 @@ def bypass(*_args: object, **_kwargs: object) -> None: assert apply_raw_authority_recovery(plan).status == "already_satisfied" +def test_index_seed_digest_avoids_unbounded_sql_parameters(tmp_path: Path) -> None: + """Candidate exclusion remains usable below SQLite's configured bind limit.""" + + initialize_active_archive_root(tmp_path) + active_index = _seed_index_seeds(tmp_path) + with sqlite3.connect(active_index) as conn: + conn.execute("ATTACH DATABASE ? AS src", (str(tmp_path / "source.db"),)) + conn.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 1) + digest = _index_seed_digest( + conn, + excluded_keys={ + "raw_revision_heads": ("k-r-present", "k-r-gone"), + "raw_revision_applications": ("d-r-present", "d-r-gone"), + }, + ) + assert len(digest) == 64 + + def test_index_prune_refuses_stale_active_pointer_and_wrong_backup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -487,6 +600,75 @@ def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, di ) +def test_index_prune_resume_accepts_append_only_successor_rows(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Receipt finalization keeps the committed candidate outcome despite later ingest successors.""" + + initialize_active_archive_root(tmp_path) + active_index = _seed_index_seeds(tmp_path) + backup = _backup_authority(tmp_path, monkeypatch, tier="index") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) + + from polylogue.maintenance import raw_authority_recovery + + original_write = raw_authority_recovery._write_durable_immutable + + def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, digest_field: str) -> Path: + if digest_field == "receipt_sha256": + raise OSError("injected final receipt write failure") + return original_write(root, path, payload, digest_field=digest_field) + + monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", fail_final_receipt) + with pytest.raises(RawAuthorityRecoveryError, match="injected final receipt write failure"): + apply_raw_authority_recovery(plan) + monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", original_write) + + with sqlite3.connect(active_index) as conn: + conn.execute( + "INSERT INTO raw_revision_applications " + "(decision_id, raw_id, session_id, logical_source_key, source_revision, acquisition_generation, " + "decision, detail, decided_at_ms) VALUES " + "('d-successor', 'r-successor', 's-successor', 'k-successor', 'sr', 1, 'selected_baseline', 'd', 2)" + ) + + recovered = resume_raw_authority_recovery( + tmp_path, + RecoveryOperation.PRUNE_INDEX_SEEDS, + operation_id=plan.operation_id, + ) + assert recovered.status == "already_satisfied" + with sqlite3.connect(active_index) as conn: + assert conn.execute( + "SELECT COUNT(*) FROM raw_revision_applications WHERE decision_id = 'd-successor'" + ).fetchone() == (1,) + + +def test_index_recovery_actuator_reports_index_targets_and_deleted_candidates( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The executor receipt describes the active-index mutation it actually performed.""" + + initialize_active_archive_root(tmp_path) + _seed_index_seeds(tmp_path) + backup = _backup_authority(tmp_path, monkeypatch, tier="index") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) + args = _RecoveryArgs( + archive_root=tmp_path, + operation=RecoveryOperation.PRUNE_INDEX_SEEDS, + operation_id=plan.operation_id, + expected_plan_digest=plan.plan_digest, + backup_manifest=backup, + receipt_path=Path(plan.receipt_path), + ) + actuator = PruneOrphanedIndexRevisionSeedsActuator() + + prepared = actuator.prepare(args) + assert prepared.target_refs == ("index:prune_orphaned_index_revision_seeds",) + assert prepared.affected_tiers == ("index",) + receipt = actuator.apply(prepared, args) + assert receipt.target_refs == ("index:prune_orphaned_index_revision_seeds",) + assert receipt.affected_count == 2 + + def test_named_compatibility_facade_requires_index_backup(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _seed_index_seeds(tmp_path) From 298f895577202fc7512ed169171d47b95e934eab Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 06:00:53 +0200 Subject: [PATCH 17/27] fix(raw-authority): retire aborted continuity intents Clear a source-continuity pending intent only after its source transaction is demonstrably rolled back, leaving it durable after any committed or ambiguous outcome. Stream protected-table fingerprints by primary key or rowid so inspection never collects an entire table in Python memory. --- .../maintenance/raw_authority_recovery.py | 42 +++++++++++---- .../maintenance/test_raw_authority_reset.py | 54 +++++++++++++++++++ 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index dc3de167a7..e6968b830d 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -50,6 +50,7 @@ from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.durable_change_train import ( DurableChangeTrainError, + clear_source_continuity_pending_intent, reconcile_durable_change_train_startup, write_source_continuity_pending_intent, ) @@ -145,13 +146,26 @@ def _value_for_digest(value: object) -> object: def _table_digest(conn: sqlite3.Connection, name: str) -> str: + """Hash one table through a deterministic, streaming row traversal.""" + quoted = _quote_identifier(name) - columns = [str(row[1]) for row in conn.execute(f"PRAGMA table_info({quoted})")] + table_info = tuple(conn.execute(f"PRAGMA table_info({quoted})")) + columns = [str(row[1]) for row in table_info] if not columns: raise RawAuthorityRecoveryError(f"cannot fingerprint missing or malformed table: {name}") - rows = [[_value_for_digest(value) for value in row] for row in conn.execute(f"SELECT * FROM {quoted}")] - rows.sort(key=_canonical_bytes) - return _digest({"name": name, "columns": columns, "rows": rows}) + primary_key_columns = [str(row[1]) for row in sorted(table_info, key=lambda row: int(row[5])) if int(row[5])] + order_by = ", ".join(_quote_identifier(column) for column in primary_key_columns) or "rowid" + digest = hashlib.sha256() + + def update(value: object) -> None: + encoded = _canonical_bytes(value) + digest.update(len(encoded).to_bytes(8, byteorder="big")) + digest.update(encoded) + + update({"name": name, "columns": columns}) + for row in conn.execute(f"SELECT * FROM {quoted} ORDER BY {order_by}"): + update([_value_for_digest(value) for value in row]) + return digest.hexdigest() def _protected_digest(conn: sqlite3.Connection, *, excluded: tuple[str, ...]) -> str: @@ -908,7 +922,7 @@ def _recovery_intent(plan: RawAuthorityRecoveryPlan) -> dict[str, object]: } -def _write_source_continuity_pending_intent(plan: RawAuthorityRecoveryPlan) -> None: +def _write_source_continuity_pending_intent(plan: RawAuthorityRecoveryPlan) -> Path: """Persist source-train refresh evidence before the reset can commit.""" authority = plan.backup_authority @@ -918,7 +932,7 @@ def _write_source_continuity_pending_intent(plan: RawAuthorityRecoveryPlan) -> N try: with closing(sqlite3.connect(f"file:{root / 'source.db'}?mode=ro", uri=True)) as conn: before = capture_durable_database_evidence(conn, ArchiveTier.SOURCE) - write_source_continuity_pending_intent( + return write_source_continuity_pending_intent( root, mutation_receipt=Path(plan.receipt_path), backup_manifest=Path(str(authority["manifest_path"])), @@ -1070,13 +1084,17 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: ) _write_recovery_intent(plan) if operation is RecoveryOperation.RESET_CENSUS: - _write_source_continuity_pending_intent(plan) + continuity_intent = _write_source_continuity_pending_intent(plan) + else: + continuity_intent = None if operation is RecoveryOperation.RESET_CENSUS: with closing(sqlite3.connect(source_db)) as conn: conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") - conn.execute("BEGIN IMMEDIATE") + transaction_started = False try: + conn.execute("BEGIN IMMEDIATE") + transaction_started = True _revalidate_common(plan, root, location) _validate_ledger(conn) _validate_integrity(conn, tier="source") @@ -1091,7 +1109,13 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) conn.commit() except Exception: - conn.rollback() + if not transaction_started or conn.in_transaction: + try: + if conn.in_transaction: + conn.rollback() + finally: + if continuity_intent is not None: + clear_source_continuity_pending_intent(continuity_intent) raise else: with closing(sqlite3.connect(index_db)) as conn: diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index a0fadc1d71..92d08cfecd 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -16,6 +16,7 @@ PruneOrphanedIndexRevisionSeedsActuator, RawAuthorityRecoveryError, RecoveryOperation, + _canonical_bytes, _index_seed_digest, _RecoveryArgs, _write_recovery_intent, @@ -288,6 +289,59 @@ def record_pending( assert not pending_paths[0].exists() +def test_census_reset_clears_continuity_intent_when_precommit_backup_revalidation_refuses( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A reset refusal cannot strand a continuity intent without its receipt.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + + def reject_changed_backup(*_args: object, **_kwargs: object) -> None: + raise RawAuthorityRecoveryError("backup authority changed before commit") + + monkeypatch.setattr("polylogue.maintenance.raw_authority_recovery._backup_from_plan", reject_changed_backup) + with pytest.raises(RawAuthorityRecoveryError, match="backup authority changed before commit"): + apply_raw_authority_recovery(plan) + + pending_root = tmp_path / ".maintenance-state" / "source-continuity-pending" + assert list(pending_root.glob("*.json")) == [] + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + + +def test_recovery_protected_digest_streams_rows_without_a_whole_table_payload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Inspection hashes protected tables incrementally instead of serializing all rows together.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("CREATE TABLE protected_rows (row_id INTEGER PRIMARY KEY, payload BLOB NOT NULL)") + conn.executemany( + "INSERT INTO protected_rows (row_id, payload) VALUES (?, ?)", + [(index, bytes([index]) * 32) for index in range(1, 5)], + ) + + def reject_whole_table_payload(value: object) -> bytes: + if isinstance(value, dict) and "rows" in value: + raise AssertionError("protected digest materialized a whole table") + return _canonical_bytes(value) + + monkeypatch.setattr("polylogue.maintenance.raw_authority_recovery._canonical_bytes", reject_whole_table_payload) + before = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS) + with sqlite3.connect(tmp_path / "source.db") as conn: + conn.execute("UPDATE protected_rows SET payload = x'ff' WHERE row_id = 2") + after = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS) + + assert after.protected_digest != before.protected_digest + + def test_recovery_receipt_path_must_be_owned_by_the_archive(tmp_path: Path) -> None: initialize_active_archive_root(tmp_path) _seed_ledger(tmp_path / "source.db") From 4ccc62b829fbcd6f36d2f28f17221556b7933cd4 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 06:29:08 +0200 Subject: [PATCH 18/27] fix(raw-authority): bound prune resume evidence Treat an intact index-prune intent as uncommitted so resume reaches the executor authorization boundary. Replace retained-row maps with streaming watermark proofs that retain only the candidate rowids already required for deletion, while allowing append-only application successors. --- .../maintenance/raw_authority_recovery.py | 229 +++++++++++++----- .../maintenance/test_raw_authority_reset.py | 64 +++++ 2 files changed, 229 insertions(+), 64 deletions(-) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index e6968b830d..87de39364f 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -79,6 +79,11 @@ class RecoveryOperation(StrEnum): "raw_authority_censuses", ) _INDEX_TARGETS = ("raw_revision_heads", "raw_revision_applications") +_INDEX_SEED_KEY_COLUMNS = { + "raw_revision_heads": "logical_source_key", + "raw_revision_applications": "decision_id", +} +_INDEX_SEED_APPEND_ONLY_TABLES = frozenset({"raw_revision_applications"}) class RawAuthorityRecoveryError(RuntimeError): @@ -299,59 +304,161 @@ def _index_candidates(conn: sqlite3.Connection) -> dict[str, tuple[str, ...]]: return {"raw_revision_heads": heads, "raw_revision_applications": applications} -def _index_seed_rows( +def _stream_index_seed_rows( conn: sqlite3.Connection, *, excluded_keys: Mapping[str, tuple[str, ...]] | None = None -) -> dict[str, dict[str, str]]: - """Return stable row digests keyed by each index-seed primary key.""" +) -> dict[str, dict[str, object]]: + """Return bounded retained-row proofs for the index-seed tables.""" - key_columns = { - "raw_revision_heads": "logical_source_key", - "raw_revision_applications": "decision_id", - } - payload: dict[str, dict[str, str]] = {} - for table, key_column in key_columns.items(): + excluded_rowids = _index_seed_candidate_rowids(conn, excluded_keys=excluded_keys) + payload: dict[str, dict[str, object]] = {} + for table in _INDEX_SEED_KEY_COLUMNS: quoted = _quote_identifier(table) columns = [str(row[1]) for row in conn.execute(f"PRAGMA table_info({quoted})")] if not columns: raise RawAuthorityRecoveryError(f"cannot fingerprint missing or malformed table: {table}") - key_index = columns.index(key_column) - excluded = set(() if excluded_keys is None else excluded_keys.get(table, ())) - row_digests: dict[str, str] = {} - for row in conn.execute(f"SELECT * FROM {quoted}"): - key = str(row[key_index]) - if key in excluded: - continue - if key in row_digests: - raise RawAuthorityRecoveryError(f"index seed table has a duplicate primary key: {table}.{key_column}") - row_digests[key] = _digest({"columns": columns, "row": [_value_for_digest(value) for value in row]}) - payload[table] = row_digests + rowid_watermark = int(conn.execute(f"SELECT COALESCE(MAX(rowid), 0) FROM {quoted}").fetchone()[0]) + row_count, rows_sha256 = _stream_index_seed_table( + conn, + table=table, + columns=columns, + excluded_rowids=excluded_rowids[table], + rowid_watermark=rowid_watermark, + ) + payload[table] = { + "excluded_rowids": list(excluded_rowids[table]), + "retained_row_count": row_count, + "retained_rows_sha256": rows_sha256, + "rowid_watermark": rowid_watermark, + } return payload +def _index_seed_candidate_rowids( + conn: sqlite3.Connection, *, excluded_keys: Mapping[str, tuple[str, ...]] | None +) -> dict[str, tuple[int, ...]]: + """Resolve the bounded candidate set to its pre-prune SQLite rowids.""" + + rowids: dict[str, tuple[int, ...]] = {} + for table, key_column in _INDEX_SEED_KEY_COLUMNS.items(): + quoted_table = _quote_identifier(table) + quoted_key = _quote_identifier(key_column) + resolved: list[int] = [] + for key in () if excluded_keys is None else excluded_keys.get(table, ()): + row = conn.execute(f"SELECT rowid FROM {quoted_table} WHERE {quoted_key} = ?", (key,)).fetchone() + if row is None: + raise RawAuthorityRecoveryError("planned index seed candidate disappeared before proof publication") + resolved.append(int(row[0])) + rowids[table] = tuple(sorted(resolved)) + return rowids + + +def _stream_index_seed_table( + conn: sqlite3.Connection, + *, + table: str, + columns: list[str], + excluded_rowids: tuple[int, ...], + rowid_watermark: int, +) -> tuple[int, str]: + """Hash retained rows through a rowid-bounded, ordered cursor.""" + + quoted = _quote_identifier(table) + excluded = set(excluded_rowids) + digest = hashlib.sha256() + + def update(value: object) -> None: + encoded = _canonical_bytes(value) + digest.update(len(encoded).to_bytes(8, byteorder="big")) + digest.update(encoded) + + update( + { + "table": table, + "columns": columns, + "excluded_rowids": list(excluded_rowids), + "rowid_watermark": rowid_watermark, + } + ) + retained_count = 0 + for row in conn.execute(f"SELECT rowid, * FROM {quoted} WHERE rowid <= ? ORDER BY rowid", (rowid_watermark,)): + if int(row[0]) in excluded: + continue + update({"rowid": int(row[0]), "row": [_value_for_digest(value) for value in row[1:]]}) + retained_count += 1 + return retained_count, digest.hexdigest() + + +def _index_seed_candidate_presence( + conn: sqlite3.Connection, candidate_keys: Mapping[str, tuple[str, ...]] +) -> tuple[int, int]: + """Return the number of planned candidates and those still present.""" + + planned = 0 + present = 0 + for table, key_column in _INDEX_SEED_KEY_COLUMNS.items(): + quoted = _quote_identifier(table) + quoted_key = _quote_identifier(key_column) + for key in candidate_keys[table]: + planned += 1 + if conn.execute(f"SELECT 1 FROM {quoted} WHERE {quoted_key} = ?", (key,)).fetchone() is not None: + present += 1 + return planned, present + + def _index_seed_digest(conn: sqlite3.Connection, *, excluded_keys: Mapping[str, tuple[str, ...]] | None = None) -> str: - """Hash a bounded, key-addressable index-seed row snapshot.""" + """Hash a bounded index-seed retained-row proof.""" - return _digest(_index_seed_rows(conn, excluded_keys=excluded_keys)) + return _digest(_stream_index_seed_rows(conn, excluded_keys=excluded_keys)) -def _verify_index_seed_post_target(conn: sqlite3.Connection, plan: RawAuthorityRecoveryPlan) -> None: - """Require planned retained rows and candidate absence, allowing append-only successors.""" +def _verify_index_seed_post_target(conn: sqlite3.Connection, plan: RawAuthorityRecoveryPlan) -> bool: + """Prove a committed prune while permitting later append-only applications.""" - if plan.post_target_rows is None or plan.post_target_digest is None: - raise RawAuthorityRecoveryError("recovery plan is missing its exact index seed post-target state") - if _digest(plan.post_target_rows) != plan.post_target_digest: - raise RawAuthorityRecoveryError("recovery plan has an invalid exact index seed post-target digest") - observed = _index_seed_rows(conn) - for table in _INDEX_TARGETS: - candidate_keys = set(plan.candidate_keys[table]) - if candidate_keys & set(observed[table]): - raise RawAuthorityRecoveryError("recovery intent still contains a planned orphaned index seed") - expected = plan.post_target_rows.get(table) + if plan.post_target_proof is None: + raise RawAuthorityRecoveryError("recovery plan is missing its bounded index seed post-target proof") + planned_candidates, present_candidates = _index_seed_candidate_presence(conn, plan.candidate_keys) + if planned_candidates == 0 or present_candidates == planned_candidates: + return False + if present_candidates: + raise RawAuthorityRecoveryError("recovery intent has only partially pruned its planned index seed candidates") + for table in _INDEX_SEED_KEY_COLUMNS: + expected = plan.post_target_proof.get(table) if not isinstance(expected, dict): - raise RawAuthorityRecoveryError("recovery plan has malformed index seed post-target rows") - retained = {key: observed[table].get(key) for key in expected} - if retained != expected: + raise RawAuthorityRecoveryError("recovery plan has malformed bounded index seed post-target proof") + rowid_watermark = expected.get("rowid_watermark") + excluded_rowids = expected.get("excluded_rowids") + expected_count = expected.get("retained_row_count") + expected_digest = expected.get("retained_rows_sha256") + if ( + not isinstance(rowid_watermark, int) + or rowid_watermark < 0 + or not isinstance(excluded_rowids, list) + or any(not isinstance(rowid, int) or rowid <= 0 for rowid in excluded_rowids) + or not isinstance(expected_count, int) + or expected_count < 0 + or not isinstance(expected_digest, str) + ): + raise RawAuthorityRecoveryError("recovery plan has malformed bounded index seed post-target proof") + columns = [str(row[1]) for row in conn.execute(f"PRAGMA table_info({_quote_identifier(table)})")] + observed_count, observed_digest = _stream_index_seed_table( + conn, + table=table, + columns=columns, + excluded_rowids=tuple(excluded_rowids), + rowid_watermark=rowid_watermark, + ) + if (observed_count, observed_digest) != (expected_count, expected_digest): raise RawAuthorityRecoveryError("recovery intent does not match the exact committed index seed state") + if table not in _INDEX_SEED_APPEND_ONLY_TABLES: + quoted_table = _quote_identifier(table) + if conn.execute(f"SELECT 1 FROM {quoted_table} WHERE rowid > ?", (rowid_watermark,)).fetchone() is not None: + raise RawAuthorityRecoveryError("recovery intent has an unexpected post-plan index seed successor") + if any( + conn.execute(f"SELECT 1 FROM {quoted_table} WHERE rowid = ?", (rowid,)).fetchone() is not None + for rowid in excluded_rowids + ): + raise RawAuthorityRecoveryError("recovery intent replaced a non-append-only index seed candidate") + return True def _validate_backup(path: Path | None, *, tier: ArchiveTier, connection: sqlite3.Connection) -> dict[str, object]: @@ -424,8 +531,7 @@ class RawAuthorityRecoveryPlan: active_generation: dict[str, object] before_counts: dict[str, int] candidate_keys: dict[str, tuple[str, ...]] - post_target_rows: dict[str, dict[str, str]] | None - post_target_digest: str | None + post_target_proof: dict[str, dict[str, object]] | None protected_digest: str backup_authority: dict[str, object] | None receipt_path: str @@ -447,8 +553,7 @@ def _payload(self) -> dict[str, object]: "active_generation": self.active_generation, "before_counts": self.before_counts, "candidate_keys": {key: list(value) for key, value in self.candidate_keys.items()}, - "post_target_rows": self.post_target_rows, - "post_target_digest": self.post_target_digest, + "post_target_proof": self.post_target_proof, "protected_digest": self.protected_digest, "backup_authority": self.backup_authority, "receipt_path": self.receipt_path, @@ -487,26 +592,26 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: "source_snapshot", "active_generation", "before_counts", - "post_target_rows", - "post_target_digest", + "post_target_proof", "protected_digest", "receipt_path", ) if any(key not in payload for key in required): raise RawAuthorityRecoveryError("raw-authority recovery plan is missing a required field") - post_target_rows = payload["post_target_rows"] - if post_target_rows is not None and ( - not isinstance(post_target_rows, dict) + post_target_proof = payload["post_target_proof"] + if post_target_proof is not None and ( + not isinstance(post_target_proof, dict) or any( - not isinstance(rows, dict) - or any(not isinstance(key, str) or not isinstance(value, str) for key, value in rows.items()) - for rows in post_target_rows.values() + not isinstance(proof, dict) + or not isinstance(proof.get("rowid_watermark"), int) + or not isinstance(proof.get("excluded_rowids"), list) + or any(not isinstance(rowid, int) or rowid <= 0 for rowid in proof.get("excluded_rowids", [])) + or not isinstance(proof.get("retained_row_count"), int) + or not isinstance(proof.get("retained_rows_sha256"), str) + for proof in post_target_proof.values() ) ): - raise RawAuthorityRecoveryError("raw-authority recovery plan post-target rows are malformed") - post_target_digest = payload["post_target_digest"] - if post_target_digest is not None and not isinstance(post_target_digest, str): - raise RawAuthorityRecoveryError("raw-authority recovery plan post-target digest is malformed") + raise RawAuthorityRecoveryError("raw-authority recovery plan post-target proof is malformed") return cls( operation_id=str(payload["operation_id"]), operation=str(payload["operation"]), @@ -521,8 +626,7 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=_int_mapping(payload["before_counts"], field="before_counts"), candidate_keys=candidates, - post_target_rows=cast(dict[str, dict[str, str]] | None, post_target_rows), - post_target_digest=post_target_digest, + post_target_proof=cast(dict[str, dict[str, object]] | None, post_target_proof), protected_digest=str(payload["protected_digest"]), backup_authority=( cast(dict[str, object], payload["backup_authority"]) @@ -793,8 +897,7 @@ def _build_plan( _validate_integrity(source, tier="source") counts = source_counts candidate_keys: dict[str, tuple[str, ...]] = {} - post_target_rows: dict[str, dict[str, str]] | None = None - post_target_digest: str | None = None + post_target_proof: dict[str, dict[str, object]] | None = None protected = _protected_digest(source, excluded=_RESET_TABLES) else: with closing(sqlite3.connect(f"file:{index_db}?mode=ro", uri=True)) as index: @@ -803,8 +906,7 @@ def _build_plan( _validate_integrity(index, tier="active index") candidate_keys = _index_candidates(index) counts = _count_tables(index, _INDEX_TARGETS) - post_target_rows = _index_seed_rows(index, excluded_keys=candidate_keys) - post_target_digest = _digest(post_target_rows) + post_target_proof = _stream_index_seed_rows(index, excluded_keys=candidate_keys) protected = _protected_digest(index, excluded=_INDEX_TARGETS) source_snapshot = source_revision_snapshot(root) backup_authority: dict[str, object] | None = None @@ -830,8 +932,7 @@ def _build_plan( "active_generation": _generation_identity(root, location), "before_counts": counts, "candidate_keys": {key: list(value) for key, value in candidate_keys.items()}, - "post_target_rows": post_target_rows, - "post_target_digest": post_target_digest, + "post_target_proof": post_target_proof, "protected_digest": protected, "backup_authority": backup_authority, "receipt_path": str(receipt), @@ -850,8 +951,7 @@ def _build_plan( active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=counts, candidate_keys=candidate_keys, - post_target_rows=post_target_rows, - post_target_digest=post_target_digest, + post_target_proof=post_target_proof, protected_digest=protected, backup_authority=backup_authority, receipt_path=str(receipt), @@ -1046,7 +1146,8 @@ def _committed_postflight(plan: RawAuthorityRecoveryPlan) -> tuple[dict[str, int after_counts = _count_tables(conn, _INDEX_TARGETS) expected_after = {key: plan.before_counts[key] - len(plan.candidate_keys[key]) for key in _INDEX_TARGETS} if operation is RecoveryOperation.PRUNE_INDEX_SEEDS: - _verify_index_seed_post_target(conn, plan) + if not _verify_index_seed_post_target(conn, plan): + return None elif after_counts != expected_after: return None postflight = _postflight(conn, protected_digest=plan.protected_digest, excluded=excluded) diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 92d08cfecd..ab9a8d601b 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -604,6 +604,34 @@ def test_index_seed_digest_avoids_unbounded_sql_parameters(tmp_path: Path) -> No assert len(digest) == 64 +def test_index_prune_plan_keeps_retained_seed_evidence_bounded(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A real recovery plan stores summary proof, never every retained seed identity.""" + + initialize_active_archive_root(tmp_path) + active_index = _seed_index_seeds(tmp_path) + with sqlite3.connect(active_index) as conn: + conn.executemany( + "INSERT INTO raw_revision_applications " + "(decision_id, raw_id, session_id, logical_source_key, source_revision, acquisition_generation, " + "decision, detail, decided_at_ms) VALUES (?, 'r-present', ?, ?, 'sr', 1, 'selected_baseline', 'd', 2)", + [(f"d-retained-{index}", f"s-retained-{index}", f"k-retained-{index}") for index in range(64)], + ) + + backup = _backup_authority(tmp_path, monkeypatch, tier="index") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) + + assert plan.post_target_proof is not None + applications_proof = plan.post_target_proof["raw_revision_applications"] + assert set(applications_proof) == { + "excluded_rowids", + "retained_row_count", + "retained_rows_sha256", + "rowid_watermark", + } + assert applications_proof["retained_row_count"] == 65 + assert "d-retained-0" not in json.dumps(plan.to_dict(), sort_keys=True) + + def test_index_prune_refuses_stale_active_pointer_and_wrong_backup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -696,6 +724,42 @@ def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, di ).fetchone() == (1,) +def test_uncommitted_index_prune_intent_reauthorizes_before_deleting_candidates( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An intact index-prune intent resumes through executor authorization, not postflight receipt recovery.""" + + initialize_active_archive_root(tmp_path) + active_index = _seed_index_seeds(tmp_path) + backup = _backup_authority(tmp_path, monkeypatch, tier="index") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) + _write_recovery_intent(plan) + + original_authorize = OperationExecutor.authorize + + def require_authorization(*_args: object, **_kwargs: object) -> Never: + raise RuntimeError("executor authorization was required") + + monkeypatch.setattr(OperationExecutor, "authorize", require_authorization) + with pytest.raises(RuntimeError, match="executor authorization was required"): + resume_raw_authority_recovery( + tmp_path, + RecoveryOperation.PRUNE_INDEX_SEEDS, + operation_id=plan.operation_id, + ) + with sqlite3.connect(active_index) as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_revision_heads").fetchone() == (2,) + assert conn.execute("SELECT COUNT(*) FROM raw_revision_applications").fetchone() == (2,) + + monkeypatch.setattr(OperationExecutor, "authorize", original_authorize) + resumed = resume_raw_authority_recovery( + tmp_path, + RecoveryOperation.PRUNE_INDEX_SEEDS, + operation_id=plan.operation_id, + ) + assert resumed.status == "applied" + + def test_index_recovery_actuator_reports_index_targets_and_deleted_candidates( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From f489e3e17375a8529c0d44387c12c0aa19117122 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 06:56:41 +0200 Subject: [PATCH 19/27] fix(raw-authority): harden continuity recovery Declare the source and index recovery targets in the operation catalog. Keep an unfinalized source receipt pending for raw-authority resume, continue other startup recovery, and require non-empty integer zero-count reset proof. --- .../maintenance/raw_authority_recovery.py | 8 ++- polylogue/operations/specs.py | 22 ++++++ .../storage/sqlite/durable_change_train.py | 10 ++- tests/unit/operations/test_specs.py | 16 +++++ .../unit/storage/test_durable_change_train.py | 69 +++++++++++++++++++ 5 files changed, 123 insertions(+), 2 deletions(-) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index 87de39364f..4429ebfd25 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -1023,7 +1023,13 @@ def _recovery_intent(plan: RawAuthorityRecoveryPlan) -> dict[str, object]: def _write_source_continuity_pending_intent(plan: RawAuthorityRecoveryPlan) -> Path: - """Persist source-train refresh evidence before the reset can commit.""" + """Persist source-train refresh evidence before the reset can commit. + + The pending intent names the future recovery receipt before the SQLite + transaction commits. If a crash occurs in that interval, startup leaves + the pending intent in place as ``not_yet_finalized`` so the matching + durable raw-authority intent can still resume and publish the receipt. + """ authority = plan.backup_authority if not isinstance(authority, dict): diff --git a/polylogue/operations/specs.py b/polylogue/operations/specs.py index 86891d8967..de58b0f5ff 100644 --- a/polylogue/operations/specs.py +++ b/polylogue/operations/specs.py @@ -969,6 +969,17 @@ def to_dict(self) -> JSONDocumentList: effects=("DbRead", "DbWrite", "Destructive"), safety_guards=("write_role_required", "confirmed_before_execute", "explicit_dry_run_evidence"), executor_status="executor-routed", + target_authority=( + TargetAuthorityPolicy( + key="raw-authority-recovery-source", + target_kinds=("source",), + required_capabilities=("archive.raw_authority_recovery",), + destructive_class="reset", + required_confirmation="confirm_flag", + allowed_durabilities=("derived",), + allowed_recovery=("none",), + ), + ), ), OperationSpec( name="mutate-prune-orphaned-index-revision-seeds", @@ -993,6 +1004,17 @@ def to_dict(self) -> JSONDocumentList: effects=("DbRead", "DbWrite", "Destructive"), safety_guards=("write_role_required", "confirmed_before_execute", "explicit_dry_run_evidence"), executor_status="executor-routed", + target_authority=( + TargetAuthorityPolicy( + key="raw-authority-recovery-index", + target_kinds=("index",), + required_capabilities=("archive.raw_authority_recovery",), + destructive_class="reset", + required_confirmation="confirm_flag", + allowed_durabilities=("derived",), + allowed_recovery=("none",), + ), + ), ), OperationSpec( name="mutate-save-saved-view", diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index d3eeac11de..9a27789de5 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -744,6 +744,8 @@ def _recover_pending_source_continuity_intents(archive_root: Path) -> None: except (DurableChangeTrainError, KeyError, TypeError, ValueError) as exc: raise DurableChangeTrainError(f"source continuity pending intent is malformed: {path}") from exc receipt_phase = _source_mutation_receipt_phase(receipt) + if receipt_phase == "not_yet_finalized": + continue if receipt_phase == "recovered_rolled_back": clear_source_continuity_pending_intent(path) continue @@ -805,6 +807,8 @@ def _source_mutation_receipt_phase(receipt_path: Path) -> str: try: raw = json.loads(receipt_path.read_text(encoding="utf-8")) + except FileNotFoundError: + return "not_yet_finalized" except json.JSONDecodeError: return _liveness_receipt_phase(receipt_path) except (OSError, UnicodeDecodeError) as exc: @@ -949,7 +953,11 @@ def _validate_raw_authority_reset_receipt( ): raise DurableChangeTrainError("source mutation receipt does not bind the named raw-authority reset") after_counts = payload.get("after_counts") - if not isinstance(after_counts, dict) or any(value != 0 for value in after_counts.values()): + if ( + not isinstance(after_counts, dict) + or not after_counts + or any(not isinstance(value, int) or isinstance(value, bool) or value != 0 for value in after_counts.values()) + ): raise DurableChangeTrainError("source mutation receipt does not prove the raw-authority ledger reset") return {"backup_manifest_sha256": backup_digest} diff --git a/tests/unit/operations/test_specs.py b/tests/unit/operations/test_specs.py index 9d2427d447..c90917093e 100644 --- a/tests/unit/operations/test_specs.py +++ b/tests/unit/operations/test_specs.py @@ -135,6 +135,22 @@ def test_runtime_operation_catalog_has_declared_surfaces_and_code_refs() -> None assert spec.code_refs +def test_raw_authority_recovery_specs_declare_their_exact_target_kinds() -> None: + """Recovery target refs remain authorized by the production operation catalog.""" + + specs = build_runtime_operation_catalog().by_name() + + reset_policy = specs["mutate-reset-raw-authority-census"].target_authority + prune_policy = specs["mutate-prune-orphaned-index-revision-seeds"].target_authority + + assert [(policy.key, policy.target_kinds) for policy in reset_policy] == [ + ("raw-authority-recovery-source", ("source",)) + ] + assert [(policy.key, policy.target_kinds) for policy in prune_policy] == [ + ("raw-authority-recovery-index", ("index",)) + ] + + def test_declared_operation_catalog_contains_runtime_and_control_plane_operations() -> None: catalog = build_declared_operation_catalog() diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index affde4465f..6898b90177 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -688,6 +688,75 @@ def test_startup_consumes_an_already_recovered_rollback_intent(tmp_path: Path) - assert not pending_path.exists() +def test_startup_skips_unfinalized_raw_authority_receipt_and_recovers_other_intents(tmp_path: Path) -> None: + """A crash before receipt publication leaves the raw recovery intent resumable.""" + + db_path = tmp_path / "source.db" + _create_current_database(db_path) + with sqlite3.connect(db_path) as connection: + before = migration_runner.capture_durable_database_evidence(connection, ArchiveTier.SOURCE) + backup_manifest = tmp_path / "backup-manifest.json" + backup_manifest.write_text("{}\n", encoding="utf-8") + missing_pending = write_source_continuity_pending_intent( + tmp_path, + mutation_receipt=tmp_path / "raw-authority-receipt.json", + backup_manifest=backup_manifest, + pre_mutation_evidence=before, + operation_id="raw-authority-recovery", + evidence_ref="proof:raw-authority-recovery", + ) + rolled_back_receipt = tmp_path / "rolled-back.jsonl" + rolled_back_receipt.write_text('{"phase": "recovered_rolled_back"}\n', encoding="utf-8") + rolled_back_pending = write_source_continuity_pending_intent( + tmp_path, + mutation_receipt=rolled_back_receipt, + backup_manifest=backup_manifest, + pre_mutation_evidence=before, + operation_id="other-recovery", + evidence_ref="proof:other-recovery", + ) + + durable_change_train_module._recover_pending_source_continuity_intents(tmp_path) + + assert missing_pending.exists() + assert not rolled_back_pending.exists() + + +@pytest.mark.parametrize("after_counts", ({}, {"raw_authority_censuses": False})) +def test_raw_authority_reset_receipt_requires_nonempty_integer_zero_counts( + tmp_path: Path, after_counts: dict[str, object] +) -> None: + """A self-hashed reset receipt must prove each ledger table reached zero.""" + + source_path = tmp_path / "source.db" + _create_current_database(source_path) + backup_manifest = tmp_path / "backup-manifest.json" + backup_manifest.write_text("{}\n", encoding="utf-8") + payload: dict[str, object] = { + "format": "polylogue.raw-authority-recovery-receipt.v1", + "operation": "reset_raw_authority_census", + "operation_id": "raw-authority-reset", + "archive_root": str(tmp_path), + "backup_authority": { + "tier": ArchiveTier.SOURCE.value, + "manifest_path": str(backup_manifest), + "manifest_sha256": hashlib.sha256(backup_manifest.read_bytes()).hexdigest(), + }, + "after_counts": after_counts, + } + payload["receipt_sha256"] = hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ).hexdigest() + + with pytest.raises(DurableChangeTrainError, match="does not prove"): + durable_change_train_module._validate_source_mutation_receipt_bytes( + json.dumps(payload).encode("utf-8"), + source_path=source_path, + backup_manifest=backup_manifest, + operation_id="raw-authority-reset", + ) + + def test_postcondition_recovery_rejects_remaining_orphans(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: db_path = tmp_path / "source.db" _create_current_database(db_path) From 849e219ca5e0b31ec179d673100e8db66f1ade82 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 07:13:49 +0200 Subject: [PATCH 20/27] fix(raw-authority): bind receipts to executor plans Keep the authorized mutation plan through recovery execution so the receipt proves the same plan hash that was authorized. Exercise the full executor authorize/execute path. --- .../maintenance/raw_authority_recovery.py | 3 +-- .../maintenance/test_raw_authority_reset.py | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index 4429ebfd25..b86fd63557 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -1320,7 +1320,6 @@ def prepare(self, args: _RecoveryArgs) -> MutationPlan: ) def apply(self, plan: MutationPlan, args: _RecoveryArgs) -> MutationReceipt: - del plan live = inspect_raw_authority_recovery( args.archive_root, args.operation, @@ -1343,7 +1342,7 @@ def apply(self, plan: MutationPlan, args: _RecoveryArgs) -> MutationReceipt: ) return MutationReceipt( operation=self.operation, - plan_hash=args.expected_plan_digest, + plan_hash=plan.plan_hash, status="applied" if report.applied else "already_satisfied", target_refs=(target_ref,), affected_count=affected_count if report.applied else 0, diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index ab9a8d601b..461f86ba65 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -760,10 +760,10 @@ def require_authorization(*_args: object, **_kwargs: object) -> Never: assert resumed.status == "applied" -def test_index_recovery_actuator_reports_index_targets_and_deleted_candidates( +def test_index_recovery_actuator_receipt_retains_authorized_plan_identity( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """The executor receipt describes the active-index mutation it actually performed.""" + """The executor receipt remains bound to the plan it authorized.""" initialize_active_archive_root(tmp_path) _seed_index_seeds(tmp_path) @@ -779,10 +779,26 @@ def test_index_recovery_actuator_reports_index_targets_and_deleted_candidates( ) actuator = PruneOrphanedIndexRevisionSeedsActuator() - prepared = actuator.prepare(args) + executor = OperationExecutor() + prepared = executor.prepare(actuator, args) assert prepared.target_refs == ("index:prune_orphaned_index_revision_seeds",) assert prepared.affected_tiers == ("index",) - receipt = actuator.apply(prepared, args) + authorization = executor.authorize( + actuator, + prepared, + actor="test:raw-authority", + role="maintenance", + capability="archive.raw_authority_recovery", + confirmation_strength="confirm_flag", + ) + receipt = executor.execute(actuator, prepared, authorization, args) + + assert authorization.plan_hash == prepared.plan_hash + assert receipt.plan_hash == prepared.plan_hash + assert receipt.plan_hash != plan.plan_digest + domain_plan = receipt.domain_receipt["plan"] + assert isinstance(domain_plan, dict) + assert domain_plan["plan_digest"] == plan.plan_digest assert receipt.target_refs == ("index:prune_orphaned_index_revision_seeds",) assert receipt.affected_count == 2 From c7c94cf9d98ec33bab785a0ba3e80932decb6fed Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 07:43:11 +0200 Subject: [PATCH 21/27] fix(raw-authority): bind clean durable recovery plans Reject dirty source builds during recovery planning and revalidation. Preserve the declared durable target class when the executor materializes a legacy actuator target. --- .../maintenance/raw_authority_recovery.py | 17 +++++++++---- polylogue/operations/mutation_transaction.py | 8 +++++-- polylogue/operations/specs.py | 2 +- .../maintenance/test_raw_authority_reset.py | 19 +++++++++++++++ tests/unit/operations/test_operation_audit.py | 24 +++++++++++++++++-- tests/unit/operations/test_specs.py | 1 + 6 files changed, 62 insertions(+), 9 deletions(-) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index b86fd63557..954a29fe05 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -98,6 +98,17 @@ def _digest(payload: object) -> str: return hashlib.sha256(_canonical_bytes(payload)).hexdigest() +def _recovery_code_sha() -> str: + """Return the immutable build identity allowed to authorize recovery.""" + + if VERSION_INFO.dirty: + raise RawAuthorityRecoveryError("recovery requires a clean build") + code_sha = VERSION_INFO.commit + if not code_sha: + raise RawAuthorityRecoveryError("recovery requires an exact build code SHA") + return code_sha + + def _file_fingerprint(path: Path) -> dict[str, object]: try: stat = path.stat() @@ -884,9 +895,7 @@ def _build_plan( if not source_db.is_file() or not index_db.is_file(): raise FileNotFoundError(source_db if not source_db.is_file() else index_db) schema_versions = _schema_versions(root, location) - code_sha = VERSION_INFO.commit - if not code_sha: - raise RawAuthorityRecoveryError("recovery requires an exact build code SHA") + code_sha = _recovery_code_sha() identity = _archive_identity(root, location) identity_digest = _digest(identity) with closing(sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)) as source: @@ -997,7 +1006,7 @@ def _same(value: object, expected: object, field: str) -> None: def _revalidate_common(plan: RawAuthorityRecoveryPlan, root: Path, location: ArchiveLocation) -> None: if str(root.resolve(strict=False)) != plan.archive_root: raise RawAuthorityRecoveryError("recovery plan names a different archive root") - _same(VERSION_INFO.commit, plan.code_sha, "code SHA") + _same(_recovery_code_sha(), plan.code_sha, "code SHA") _same(_schema_versions(root, location), plan.schema_versions, "schema versions") _same(_archive_identity(root, location), plan.archive_identity, "archive identity") _same(_digest(plan.archive_identity), plan.archive_identity_digest, "archive identity digest") diff --git a/polylogue/operations/mutation_transaction.py b/polylogue/operations/mutation_transaction.py index 7ea937ee4a..202bdf4911 100644 --- a/polylogue/operations/mutation_transaction.py +++ b/polylogue/operations/mutation_transaction.py @@ -718,6 +718,10 @@ def _typed_plan_from_actuator( default = next(iter(policies.values()), None) if default is None: raise MutationTransactionError(f"{binding.spec.name!r} has no target authority policy") + if len(default.allowed_durabilities) != 1 or len(default.allowed_recovery) != 1: + raise MutationTransactionError( + f"{binding.spec.name!r} must emit typed targets for an ambiguous default authority policy" + ) targets = tuple( MutationTarget( kind=ref.split(":", 1)[0], @@ -725,8 +729,8 @@ def _typed_plan_from_actuator( policy_key=default.key, identity_digest=_sha256_document({"ref": ref}), effect_identity=f"{binding.spec.name}:{ref}", - durability="derived", - recovery="none", + durability=default.allowed_durabilities[0], + recovery=default.allowed_recovery[0], ) for ref in plan.target_refs ) diff --git a/polylogue/operations/specs.py b/polylogue/operations/specs.py index de58b0f5ff..3abb4912d1 100644 --- a/polylogue/operations/specs.py +++ b/polylogue/operations/specs.py @@ -976,7 +976,7 @@ def to_dict(self) -> JSONDocumentList: required_capabilities=("archive.raw_authority_recovery",), destructive_class="reset", required_confirmation="confirm_flag", - allowed_durabilities=("derived",), + allowed_durabilities=("durable",), allowed_recovery=("none",), ), ), diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 461f86ba65..dcac6854b5 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -33,6 +33,7 @@ from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root from polylogue.storage.sqlite.durable_change_train import write_source_continuity_pending_intent from polylogue.storage.sqlite.migration_runner import DurableDatabaseEvidence +from polylogue.version import VERSION_INFO def _seed_ledger(source_db: Path) -> None: @@ -172,6 +173,24 @@ def bypass(*_args: object, **_kwargs: object) -> None: apply_raw_authority_recovery(plan) +def test_census_reset_refuses_a_build_dirtied_after_planning(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """An apply cannot bind a plan after its source build changed.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + monkeypatch.setattr(VERSION_INFO, "dirty", False) + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + + monkeypatch.setattr(VERSION_INFO, "dirty", True) + with pytest.raises(RawAuthorityRecoveryError, match="clean build"): + apply_raw_authority_recovery(plan) + + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + + def test_census_reset_preserves_recovery_evidence_when_final_receipt_write_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/operations/test_operation_audit.py b/tests/unit/operations/test_operation_audit.py index e7f3bb1d3e..f55dda476d 100644 --- a/tests/unit/operations/test_operation_audit.py +++ b/tests/unit/operations/test_operation_audit.py @@ -18,6 +18,7 @@ OperationExecutor, PlanStaleError, TargetAuthorityPolicy, + TargetDurability, build_plan, ) from polylogue.operations.specs import OperationKind, OperationSpec @@ -59,7 +60,9 @@ def apply(self, plan: MutationPlan, _args: object) -> MutationReceipt: ) -def _binding(actuator: _Actuator) -> OperationBinding[object, object]: +def _binding( + actuator: _Actuator, *, target_durability: TargetDurability = "derived" +) -> OperationBinding[object, object]: spec = OperationSpec( name="mutate-fixture", kind=OperationKind.MAINTENANCE, @@ -74,7 +77,7 @@ def _binding(actuator: _Actuator) -> OperationBinding[object, object]: required_capabilities=("archive.fixture.write",), destructive_class="reversible", required_confirmation="role_only", - allowed_durabilities=("derived",), + allowed_durabilities=(target_durability,), allowed_recovery=("none",), ), ), @@ -114,6 +117,23 @@ def test_token_is_digest_only_and_consumption_run_attempt_are_atomic(tmp_path: P assert actuator.calls == 1 +def test_prepare_bound_uses_the_declared_durable_target_for_legacy_actuators() -> None: + """Fallback target construction cannot downgrade a durable policy to derived.""" + + actuator = _Actuator() + preview = OperationExecutor().prepare_bound( + _binding(actuator, target_durability="durable"), + object(), + _principal(), + archive_instance_id="archive:test", + archive_identity_digest="identity:test", + parameter_digest="params:test", + ) + + assert preview.plan.targets[0].durability == "durable" + assert preview.plan.targets[0].recovery == "none" + + def test_invalid_capability_and_stale_preview_refuse_before_apply(tmp_path: Path) -> None: audit = AuditRepository(tmp_path / "audit.db") actuator = _Actuator() diff --git a/tests/unit/operations/test_specs.py b/tests/unit/operations/test_specs.py index c90917093e..79e1909b74 100644 --- a/tests/unit/operations/test_specs.py +++ b/tests/unit/operations/test_specs.py @@ -146,6 +146,7 @@ def test_raw_authority_recovery_specs_declare_their_exact_target_kinds() -> None assert [(policy.key, policy.target_kinds) for policy in reset_policy] == [ ("raw-authority-recovery-source", ("source",)) ] + assert reset_policy[0].allowed_durabilities == ("durable",) assert [(policy.key, policy.target_kinds) for policy in prune_policy] == [ ("raw-authority-recovery-index", ("index",)) ] From afcaa468095f3c7cece81f05d8df27c02d55c86d Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 08:06:33 +0200 Subject: [PATCH 22/27] fix(raw-authority): serialize recovery continuity Preserve bounded recovery proofs across source-backed successor heads, distinguish reset receipt finalization from missing liveness evidence, and block overlapping source mutations. --- .../maintenance/raw_authority_recovery.py | 23 +++++++-- .../storage/sqlite/durable_change_train.py | 31 +++++++++-- .../maintenance/test_raw_authority_reset.py | 51 +++++++++++++++++-- .../unit/storage/test_durable_change_train.py | 25 +++++++++ 4 files changed, 119 insertions(+), 11 deletions(-) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index 954a29fe05..cc1d748d0e 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -50,6 +50,7 @@ from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.durable_change_train import ( DurableChangeTrainError, + assert_source_continuity_apply_allowed, clear_source_continuity_pending_intent, reconcile_durable_change_train_startup, write_source_continuity_pending_intent, @@ -83,7 +84,7 @@ class RecoveryOperation(StrEnum): "raw_revision_heads": "logical_source_key", "raw_revision_applications": "decision_id", } -_INDEX_SEED_APPEND_ONLY_TABLES = frozenset({"raw_revision_applications"}) +_INDEX_SEED_SUCCESSOR_TABLES = frozenset({"raw_revision_heads", "raw_revision_applications"}) class RawAuthorityRecoveryError(RuntimeError): @@ -423,7 +424,7 @@ def _index_seed_digest(conn: sqlite3.Connection, *, excluded_keys: Mapping[str, def _verify_index_seed_post_target(conn: sqlite3.Connection, plan: RawAuthorityRecoveryPlan) -> bool: - """Prove a committed prune while permitting later append-only applications.""" + """Prove a committed prune while permitting legitimate post-plan seed rows.""" if plan.post_target_proof is None: raise RawAuthorityRecoveryError("recovery plan is missing its bounded index seed post-target proof") @@ -460,7 +461,17 @@ def _verify_index_seed_post_target(conn: sqlite3.Connection, plan: RawAuthorityR ) if (observed_count, observed_digest) != (expected_count, expected_digest): raise RawAuthorityRecoveryError("recovery intent does not match the exact committed index seed state") - if table not in _INDEX_SEED_APPEND_ONLY_TABLES: + if table == "raw_revision_heads": + missing_source = conn.execute( + "SELECT 1 FROM raw_revision_heads AS h " + "WHERE h.rowid > ? " + "AND NOT EXISTS (SELECT 1 FROM src.raw_sessions AS r WHERE r.raw_id = h.accepted_raw_id) " + "LIMIT 1", + (rowid_watermark,), + ).fetchone() + if missing_source is not None: + raise RawAuthorityRecoveryError("recovery intent has an unbacked post-plan index head successor") + if table not in _INDEX_SEED_SUCCESSOR_TABLES: quoted_table = _quote_identifier(table) if conn.execute(f"SELECT 1 FROM {quoted_table} WHERE rowid > ?", (rowid_watermark,)).fetchone() is not None: raise RawAuthorityRecoveryError("recovery intent has an unexpected post-plan index seed successor") @@ -1054,6 +1065,7 @@ def _write_source_continuity_pending_intent(plan: RawAuthorityRecoveryPlan) -> P pre_mutation_evidence=before, operation_id=plan.operation_id, evidence_ref=f"proof:raw-authority-recovery:{plan.plan_digest}", + mutation_kind="raw_authority_recovery", ) except (DurableChangeTrainError, KeyError, OSError, sqlite3.Error) as exc: raise RawAuthorityRecoveryError(f"could not persist source continuity recovery intent: {exc}") from exc @@ -1198,6 +1210,11 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: after_counts=after_counts, postflight=postflight, ) + if operation is RecoveryOperation.RESET_CENSUS: + try: + assert_source_continuity_apply_allowed(root, allowed_pending_operation_id=plan.operation_id) + except DurableChangeTrainError as exc: + raise RawAuthorityRecoveryError(str(exc)) from exc _write_recovery_intent(plan) if operation is RecoveryOperation.RESET_CENSUS: continuity_intent = _write_source_continuity_pending_intent(plan) diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index 9a27789de5..e069f85e69 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -16,7 +16,7 @@ from dataclasses import dataclass, replace from importlib import resources from pathlib import Path -from typing import Final, cast +from typing import Final, Literal, cast from polylogue.storage.blob_ref_liveness import ( BlobRefLivenessCandidate, @@ -74,6 +74,7 @@ _MIGRATION_NAME_RE = re.compile(r"^(?P\d{3,})_[a-z0-9_]+\.sql$") _DROP_SQL_RE = re.compile(r"(?is)\bDROP\s+(?:TABLE|INDEX|TRIGGER|VIEW)\b") _SOURCE_CONTINUITY_PENDING_FORMAT = "polylogue.source-continuity-pending.v1" +_SourceContinuityMutationKind = Literal["blob_ref_liveness", "raw_authority_recovery"] _FRESH_DURABLE_BOOTSTRAP_FORMAT = "polylogue.durable-bootstrap.v1" _FRESH_DURABLE_BOOTSTRAP_MARKER = ".bootstrap" _FRESH_DURABLE_BOOTSTRAP_PENDING_MARKER = ".bootstrap.pending" @@ -553,6 +554,7 @@ def write_source_continuity_pending_intent( pre_mutation_evidence: DurableDatabaseEvidence, operation_id: str, evidence_ref: str, + mutation_kind: _SourceContinuityMutationKind = "blob_ref_liveness", ) -> Path: """Persist the recovery input before a source mutation can commit.""" mutation_receipt = mutation_receipt.resolve() @@ -568,6 +570,7 @@ def write_source_continuity_pending_intent( "backup_manifest": str(backup_manifest), "operation_id": operation_id, "evidence_ref": evidence_ref, + "mutation_kind": mutation_kind, "source_before": _migration_runner._manifest_json_value(pre_mutation_evidence), } pending_digest = _canonical_json_sha256(payload) @@ -658,11 +661,19 @@ def clear_source_continuity_pending_intent(path: Path) -> None: _migration_runner._fsync_manifest_directory(path.parent) -def assert_source_continuity_apply_allowed(archive_root: Path) -> None: +def assert_source_continuity_apply_allowed( + archive_root: Path, *, allowed_pending_operation_id: str | None = None +) -> None: """Reject a new source mutation that could invalidate continuity recovery.""" archive_root = archive_root.resolve() pending_root = archive_root / ".maintenance-state" / "source-continuity-pending" pending_intents = tuple(sorted(pending_root.glob("*.json"))) if pending_root.is_dir() else () + if allowed_pending_operation_id is not None: + pending_intents = tuple( + path + for path in pending_intents + if _load_source_continuity_pending_intent(path).get("operation_id") != allowed_pending_operation_id + ) if pending_intents: raise DurableChangeTrainError("source liveness apply is blocked while source continuity recovery is pending") @@ -741,9 +752,15 @@ def _recover_pending_source_continuity_intents(archive_root: Path) -> None: backup = Path(str(raw["backup_manifest"])) operation_id = str(raw["operation_id"]) evidence_ref = str(raw["evidence_ref"]) + mutation_kind = raw.get("mutation_kind", "blob_ref_liveness") + if mutation_kind not in {"blob_ref_liveness", "raw_authority_recovery"}: + raise ValueError("mutation_kind is unsupported") except (DurableChangeTrainError, KeyError, TypeError, ValueError) as exc: raise DurableChangeTrainError(f"source continuity pending intent is malformed: {path}") from exc - receipt_phase = _source_mutation_receipt_phase(receipt) + receipt_phase = _source_mutation_receipt_phase( + receipt, + allow_unfinalized_raw_authority=mutation_kind == "raw_authority_recovery", + ) if receipt_phase == "not_yet_finalized": continue if receipt_phase == "recovered_rolled_back": @@ -802,13 +819,17 @@ def validate_recovered_postcondition(pending_path: Path = path) -> None: clear_source_continuity_pending_intent(path) -def _source_mutation_receipt_phase(receipt_path: Path) -> str: +def _source_mutation_receipt_phase(receipt_path: Path, *, allow_unfinalized_raw_authority: bool) -> str: """Classify a committed source-maintenance receipt without guessing its format.""" try: raw = json.loads(receipt_path.read_text(encoding="utf-8")) except FileNotFoundError: - return "not_yet_finalized" + if allow_unfinalized_raw_authority: + return "not_yet_finalized" + raise DurableChangeTrainError( + f"source continuity pending liveness receipt is missing: {receipt_path}" + ) from None except json.JSONDecodeError: return _liveness_receipt_phase(receipt_path) except (OSError, UnicodeDecodeError) as exc: diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index dcac6854b5..caa871cdf4 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -31,8 +31,9 @@ ) from polylogue.operations.mutation_transaction import OperationExecutor from polylogue.storage.sqlite.archive_tiers.bootstrap import initialize_active_archive_root +from polylogue.storage.sqlite.archive_tiers.types import ArchiveTier from polylogue.storage.sqlite.durable_change_train import write_source_continuity_pending_intent -from polylogue.storage.sqlite.migration_runner import DurableDatabaseEvidence +from polylogue.storage.sqlite.migration_runner import DurableDatabaseEvidence, capture_durable_database_evidence from polylogue.version import VERSION_INFO @@ -701,10 +702,13 @@ def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, di ) -def test_index_prune_resume_accepts_append_only_successor_rows(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Receipt finalization keeps the committed candidate outcome despite later ingest successors.""" +def test_index_prune_resume_accepts_source_backed_successor_heads( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Receipt finalization admits later source-backed heads beyond the proof watermark.""" initialize_active_archive_root(tmp_path) + monkeypatch.setattr(VERSION_INFO, "dirty", False) active_index = _seed_index_seeds(tmp_path) backup = _backup_authority(tmp_path, monkeypatch, tier="index") plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) @@ -723,7 +727,15 @@ def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, di apply_raw_authority_recovery(plan) monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", original_write) + _seed_raw(tmp_path / "source.db", "r-successor") with sqlite3.connect(active_index) as conn: + conn.execute( + "INSERT INTO raw_revision_heads (logical_source_key, session_id, accepted_raw_id, " + "accepted_source_revision, accepted_content_hash, accepted_frontier_kind, accepted_frontier, " + "acquisition_generation, decided_at_ms) VALUES " + "('k-successor', 's-successor', 'r-successor', 'sr', ?, 'byte', 1, 0, 2)", + (bytes.fromhex("03" * 32),), + ) conn.execute( "INSERT INTO raw_revision_applications " "(decision_id, raw_id, session_id, logical_source_key, source_revision, acquisition_generation, " @@ -738,11 +750,44 @@ def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, di ) assert recovered.status == "already_satisfied" with sqlite3.connect(active_index) as conn: + assert conn.execute( + "SELECT COUNT(*) FROM raw_revision_heads WHERE logical_source_key = 'k-successor'" + ).fetchone() == (1,) assert conn.execute( "SELECT COUNT(*) FROM raw_revision_applications WHERE decision_id = 'd-successor'" ).fetchone() == (1,) +def test_census_reset_refuses_a_competing_source_continuity_intent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A reset never overlaps another source mutation's recovery evidence.""" + + initialize_active_archive_root(tmp_path) + _seed_ledger(tmp_path / "source.db") + _seed_raw(tmp_path / "source.db", "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + monkeypatch.setattr(VERSION_INFO, "dirty", False) + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + with sqlite3.connect(tmp_path / "source.db") as conn: + before = capture_durable_database_evidence(conn, ArchiveTier.SOURCE) + pending = write_source_continuity_pending_intent( + tmp_path, + mutation_receipt=tmp_path / "liveness-receipt.jsonl", + backup_manifest=backup, + pre_mutation_evidence=before, + operation_id="other-source-mutation", + evidence_ref="proof:other-source-mutation", + ) + + with pytest.raises(RawAuthorityRecoveryError, match="continuity recovery is pending"): + apply_raw_authority_recovery(plan) + + assert pending.exists() + with sqlite3.connect(tmp_path / "source.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + + def test_uncommitted_index_prune_intent_reauthorizes_before_deleting_candidates( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index 6898b90177..acac029f07 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -704,6 +704,7 @@ def test_startup_skips_unfinalized_raw_authority_receipt_and_recovers_other_inte pre_mutation_evidence=before, operation_id="raw-authority-recovery", evidence_ref="proof:raw-authority-recovery", + mutation_kind="raw_authority_recovery", ) rolled_back_receipt = tmp_path / "rolled-back.jsonl" rolled_back_receipt.write_text('{"phase": "recovered_rolled_back"}\n', encoding="utf-8") @@ -722,6 +723,30 @@ def test_startup_skips_unfinalized_raw_authority_receipt_and_recovers_other_inte assert not rolled_back_pending.exists() +def test_startup_rejects_a_missing_liveness_receipt(tmp_path: Path) -> None: + """Liveness pending evidence is corrupt when its prewritten receipt disappears.""" + + db_path = tmp_path / "source.db" + _create_current_database(db_path) + with sqlite3.connect(db_path) as connection: + before = migration_runner.capture_durable_database_evidence(connection, ArchiveTier.SOURCE) + backup_manifest = tmp_path / "backup-manifest.json" + backup_manifest.write_text("{}\n", encoding="utf-8") + pending = write_source_continuity_pending_intent( + tmp_path, + mutation_receipt=tmp_path / "missing-liveness-receipt.jsonl", + backup_manifest=backup_manifest, + pre_mutation_evidence=before, + operation_id="liveness-operation", + evidence_ref="proof:blob-ref-liveness:liveness-operation", + ) + + with pytest.raises(DurableChangeTrainError, match="liveness receipt is missing"): + durable_change_train_module._recover_pending_source_continuity_intents(tmp_path) + + assert pending.exists() + + @pytest.mark.parametrize("after_counts", ({}, {"raw_authority_censuses": False})) def test_raw_authority_reset_receipt_requires_nonempty_integer_zero_counts( tmp_path: Path, after_counts: dict[str, object] From 333eac9b5b6a247332519db5a94befff8a7e5297 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 08:16:27 +0200 Subject: [PATCH 23/27] test(raw-authority): bind continuity intent kind Keep the pre-commit continuity-intent interception aligned with the typed mutation kind and assert that census reset emits raw_authority_recovery. --- tests/unit/maintenance/test_raw_authority_reset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index caa871cdf4..6536ce4f9f 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -8,7 +8,7 @@ import shutil import sqlite3 from pathlib import Path -from typing import Never +from typing import Literal, Never import pytest @@ -286,9 +286,11 @@ def record_pending( pre_mutation_evidence: DurableDatabaseEvidence, operation_id: str, evidence_ref: str, + mutation_kind: Literal["blob_ref_liveness", "raw_authority_recovery"], ) -> Path: with sqlite3.connect(tmp_path / "source.db") as conn: assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) + assert mutation_kind == "raw_authority_recovery" pending = original_write_pending( archive_root, mutation_receipt=mutation_receipt, @@ -296,6 +298,7 @@ def record_pending( pre_mutation_evidence=pre_mutation_evidence, operation_id=operation_id, evidence_ref=evidence_ref, + mutation_kind=mutation_kind, ) pending_paths.append(pending) assert pending.is_file() From 13d86c7604b8175cc1bb811b3877337e8a358b1e Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 08:33:21 +0200 Subject: [PATCH 24/27] fix(raw-authority): bind recovery snapshots Bind census reset plans to WAL-visible ledger rows, admit source-backed head supersession during receipt recovery, and defer stale source-train validation until an interrupted reset receipt is finalized. --- .../maintenance/raw_authority_recovery.py | 64 +++++++----- .../storage/sqlite/durable_change_train.py | 14 ++- .../maintenance/test_raw_authority_reset.py | 98 ++++++++++++++++++- .../unit/storage/test_durable_change_train.py | 55 ++++++++++- 4 files changed, 198 insertions(+), 33 deletions(-) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index cc1d748d0e..1078a5f9f3 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -84,7 +84,6 @@ class RecoveryOperation(StrEnum): "raw_revision_heads": "logical_source_key", "raw_revision_applications": "decision_id", } -_INDEX_SEED_SUCCESSOR_TABLES = frozenset({"raw_revision_heads", "raw_revision_applications"}) class RawAuthorityRecoveryError(RuntimeError): @@ -194,6 +193,12 @@ def _protected_digest(conn: sqlite3.Connection, *, excluded: tuple[str, ...]) -> return _digest({name: _table_digest(conn, name) for name in tables}) +def _ledger_digest(conn: sqlite3.Connection) -> str: + """Bind census reset plans to the SQL-visible rows they will delete.""" + + return _digest({name: _table_digest(conn, name) for name in _RESET_TABLES}) + + def _schema_versions(root: Path, location: ArchiveLocation) -> dict[str, int]: paths = { ArchiveTier.SOURCE.value: root / "source.db", @@ -451,6 +456,15 @@ def _verify_index_seed_post_target(conn: sqlite3.Connection, plan: RawAuthorityR or not isinstance(expected_digest, str) ): raise RawAuthorityRecoveryError("recovery plan has malformed bounded index seed post-target proof") + if table == "raw_revision_heads": + missing_source = conn.execute( + "SELECT 1 FROM raw_revision_heads AS h " + "WHERE NOT EXISTS (SELECT 1 FROM src.raw_sessions AS r WHERE r.raw_id = h.accepted_raw_id) " + "LIMIT 1", + ).fetchone() + if missing_source is not None: + raise RawAuthorityRecoveryError("recovery intent has an unbacked index head") + continue columns = [str(row[1]) for row in conn.execute(f"PRAGMA table_info({_quote_identifier(table)})")] observed_count, observed_digest = _stream_index_seed_table( conn, @@ -461,25 +475,6 @@ def _verify_index_seed_post_target(conn: sqlite3.Connection, plan: RawAuthorityR ) if (observed_count, observed_digest) != (expected_count, expected_digest): raise RawAuthorityRecoveryError("recovery intent does not match the exact committed index seed state") - if table == "raw_revision_heads": - missing_source = conn.execute( - "SELECT 1 FROM raw_revision_heads AS h " - "WHERE h.rowid > ? " - "AND NOT EXISTS (SELECT 1 FROM src.raw_sessions AS r WHERE r.raw_id = h.accepted_raw_id) " - "LIMIT 1", - (rowid_watermark,), - ).fetchone() - if missing_source is not None: - raise RawAuthorityRecoveryError("recovery intent has an unbacked post-plan index head successor") - if table not in _INDEX_SEED_SUCCESSOR_TABLES: - quoted_table = _quote_identifier(table) - if conn.execute(f"SELECT 1 FROM {quoted_table} WHERE rowid > ?", (rowid_watermark,)).fetchone() is not None: - raise RawAuthorityRecoveryError("recovery intent has an unexpected post-plan index seed successor") - if any( - conn.execute(f"SELECT 1 FROM {quoted_table} WHERE rowid = ?", (rowid,)).fetchone() is not None - for rowid in excluded_rowids - ): - raise RawAuthorityRecoveryError("recovery intent replaced a non-append-only index seed candidate") return True @@ -552,6 +547,7 @@ class RawAuthorityRecoveryPlan: source_snapshot: str active_generation: dict[str, object] before_counts: dict[str, int] + ledger_digest: str | None candidate_keys: dict[str, tuple[str, ...]] post_target_proof: dict[str, dict[str, object]] | None protected_digest: str @@ -574,6 +570,7 @@ def _payload(self) -> dict[str, object]: "source_snapshot": self.source_snapshot, "active_generation": self.active_generation, "before_counts": self.before_counts, + "ledger_digest": self.ledger_digest, "candidate_keys": {key: list(value) for key, value in self.candidate_keys.items()}, "post_target_proof": self.post_target_proof, "protected_digest": self.protected_digest, @@ -601,6 +598,9 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: str(key): tuple(str(value) for value in cast(list[object], value_list)) for key, value_list in candidates_raw.items() } + ledger_digest = payload.get("ledger_digest") + if ledger_digest is not None and not isinstance(ledger_digest, str): + raise RawAuthorityRecoveryError("raw-authority recovery plan ledger digest is malformed") required = ( "operation_id", "operation", @@ -647,6 +647,7 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: source_snapshot=str(payload["source_snapshot"]), active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=_int_mapping(payload["before_counts"], field="before_counts"), + ledger_digest=cast(str | None, ledger_digest), candidate_keys=candidates, post_target_proof=cast(dict[str, dict[str, object]] | None, post_target_proof), protected_digest=str(payload["protected_digest"]), @@ -916,6 +917,7 @@ def _build_plan( _validate_ledger(source) _validate_integrity(source, tier="source") counts = source_counts + ledger_digest = _ledger_digest(source) candidate_keys: dict[str, tuple[str, ...]] = {} post_target_proof: dict[str, dict[str, object]] | None = None protected = _protected_digest(source, excluded=_RESET_TABLES) @@ -926,6 +928,7 @@ def _build_plan( _validate_integrity(index, tier="active index") candidate_keys = _index_candidates(index) counts = _count_tables(index, _INDEX_TARGETS) + ledger_digest = None post_target_proof = _stream_index_seed_rows(index, excluded_keys=candidate_keys) protected = _protected_digest(index, excluded=_INDEX_TARGETS) source_snapshot = source_revision_snapshot(root) @@ -951,6 +954,7 @@ def _build_plan( "source_snapshot": source_snapshot, "active_generation": _generation_identity(root, location), "before_counts": counts, + "ledger_digest": ledger_digest, "candidate_keys": {key: list(value) for key, value in candidate_keys.items()}, "post_target_proof": post_target_proof, "protected_digest": protected, @@ -970,6 +974,7 @@ def _build_plan( source_snapshot=source_snapshot, active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=counts, + ledger_digest=ledger_digest, candidate_keys=candidate_keys, post_target_proof=post_target_proof, protected_digest=protected, @@ -1014,7 +1019,13 @@ def _same(value: object, expected: object, field: str) -> None: raise RawAuthorityRecoveryError(f"recovery plan is stale: {field} changed") -def _revalidate_common(plan: RawAuthorityRecoveryPlan, root: Path, location: ArchiveLocation) -> None: +def _revalidate_common( + plan: RawAuthorityRecoveryPlan, + root: Path, + location: ArchiveLocation, + *, + source_connection: sqlite3.Connection | None = None, +) -> None: if str(root.resolve(strict=False)) != plan.archive_root: raise RawAuthorityRecoveryError("recovery plan names a different archive root") _same(_recovery_code_sha(), plan.code_sha, "code SHA") @@ -1025,6 +1036,15 @@ def _revalidate_common(plan: RawAuthorityRecoveryPlan, root: Path, location: Arc _same(_file_fingerprint(root / "source.db"), plan.source_fingerprint, "source database") _same(_file_fingerprint(location.active_index_path), plan.index_fingerprint, "active index database") _same(source_revision_snapshot(root), plan.source_snapshot, "source snapshot") + if RecoveryOperation(plan.operation) is RecoveryOperation.RESET_CENSUS: + if plan.ledger_digest is None: + raise RawAuthorityRecoveryError("recovery plan does not bind a census ledger snapshot") + if source_connection is None: + with closing(sqlite3.connect(f"file:{root / 'source.db'}?mode=ro", uri=True)) as source: + observed_ledger_digest = _ledger_digest(source) + else: + observed_ledger_digest = _ledger_digest(source_connection) + _same(observed_ledger_digest, plan.ledger_digest, "census ledger snapshot") def _recovery_intent(plan: RawAuthorityRecoveryPlan) -> dict[str, object]: @@ -1228,7 +1248,7 @@ def _apply_plan(plan: RawAuthorityRecoveryPlan) -> RawAuthorityRecoveryReport: try: conn.execute("BEGIN IMMEDIATE") transaction_started = True - _revalidate_common(plan, root, location) + _revalidate_common(plan, root, location, source_connection=conn) _validate_ledger(conn) _validate_integrity(conn, tier="source") _same(_count_tables(conn, _RESET_TABLES), before_counts, "census ledger counts") diff --git a/polylogue/storage/sqlite/durable_change_train.py b/polylogue/storage/sqlite/durable_change_train.py index e069f85e69..25cf0eab15 100644 --- a/polylogue/storage/sqlite/durable_change_train.py +++ b/polylogue/storage/sqlite/durable_change_train.py @@ -718,11 +718,13 @@ def assert_source_continuity_apply_allowed( _verify_released_train_live_tier(archive_root, connection, released[0]) -def _recover_pending_source_continuity_intents(archive_root: Path) -> None: +def _recover_pending_source_continuity_intents(archive_root: Path) -> frozenset[ArchiveTier]: """Finish committed source mutations whose manifest refresh was interrupted.""" + + deferred_tiers: set[ArchiveTier] = set() pending_root = archive_root / ".maintenance-state" / "source-continuity-pending" if not pending_root.is_dir(): - return + return frozenset() for path in sorted(pending_root.glob("*.json")): raw = _load_source_continuity_pending_intent(path) terminal = raw.get("terminal_outcome") @@ -762,6 +764,7 @@ def _recover_pending_source_continuity_intents(archive_root: Path) -> None: allow_unfinalized_raw_authority=mutation_kind == "raw_authority_recovery", ) if receipt_phase == "not_yet_finalized": + deferred_tiers.add(ArchiveTier.SOURCE) continue if receipt_phase == "recovered_rolled_back": clear_source_continuity_pending_intent(path) @@ -817,6 +820,7 @@ def validate_recovered_postcondition(pending_path: Path = path) -> None: mark_source_continuity_pending_intent_terminal(path, error=exc) else: clear_source_continuity_pending_intent(path) + return frozenset(deferred_tiers) def _source_mutation_receipt_phase(receipt_path: Path, *, allow_unfinalized_raw_authority: bool) -> str: @@ -2274,7 +2278,7 @@ def _reconcile_durable_change_train_startup_locked( live_evidence_cache: dict[ArchiveTier, _DurableForwardVersionEvidence] | None = None, ) -> tuple[Path, ...]: """Reconcile persisted trains while the caller holds archive ownership.""" - _recover_pending_source_continuity_intents(archive_root) + deferred_tiers = _recover_pending_source_continuity_intents(archive_root) manifest_root = archive_root / ".maintenance-state" / "durable-change-trains" reconciled: list[Path] = [] live_evidence_by_tier: dict[ArchiveTier, DurableDatabaseEvidence] = {} @@ -2339,6 +2343,8 @@ def record_reconciled(path: Path) -> None: # Recovery runs first so an indeterminate persisted failure keeps its # stronger fail-closed error rather than being masked by chain validation. for tier, adoption_floor in DURABLE_MIGRATION_ADOPTION_FLOORS.items(): + if tier in deferred_tiers: + continue tier_path = archive_root / f"{tier.value}.db" if not tier_path.is_file(): continue @@ -2368,6 +2374,8 @@ def record_reconciled(path: Path) -> None: train = load_durable_change_train_manifest(manifest_path) if train.state is not DurableChangeTrainState.RELEASED: continue + if train.tier in deferred_tiers: + continue with _open_existing_tier(archive_root / f"{train.tier.value}.db") as live: actual = live_evidence_by_tier.get(train.tier) if actual is None: diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index 6536ce4f9f..a76838130f 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -192,6 +192,32 @@ def test_census_reset_refuses_a_build_dirtied_after_planning(tmp_path: Path, mon assert conn.execute("SELECT COUNT(*) FROM raw_authority_censuses").fetchone() == (1,) +def test_census_reset_refuses_wal_visible_ledger_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A changed ledger row invalidates a reset plan even when its main file is unchanged.""" + + initialize_active_archive_root(tmp_path) + source_db = tmp_path / "source.db" + _seed_ledger(source_db) + _seed_raw(source_db, "r-keep") + backup = _backup_authority(tmp_path, monkeypatch, tier="source") + with sqlite3.connect(source_db) as conn: + assert conn.execute("PRAGMA journal_mode = WAL").fetchone() == ("wal",) + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + main_database_before = source_db.read_bytes() + + with sqlite3.connect(source_db) as conn: + conn.execute("UPDATE raw_authority_censuses SET residual_json = '{\"changed\":true}' WHERE census_id = 'c1'") + + assert source_db.read_bytes() == main_database_before + assert source_db.with_name("source.db-wal").is_file() + with pytest.raises(RawAuthorityRecoveryError, match="census ledger snapshot"): + apply_raw_authority_recovery(plan) + with sqlite3.connect(source_db) as conn: + assert conn.execute("SELECT residual_json FROM raw_authority_censuses WHERE census_id = 'c1'").fetchone() == ( + '{"changed":true}', + ) + + def test_census_reset_preserves_recovery_evidence_when_final_receipt_write_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -672,8 +698,8 @@ def test_index_prune_refuses_stale_active_pointer_and_wrong_backup( apply_raw_authority_recovery(plan, backup_manifest=tmp_path / "different.json") -def test_index_prune_resume_refuses_changed_retained_seed_rows(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Receipt finalization requires the exact retained target rows from the committed prune.""" +def test_index_prune_resume_refuses_an_unbacked_retained_head(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Receipt finalization refuses a retained head whose raw source no longer exists.""" initialize_active_archive_root(tmp_path) active_index = _seed_index_seeds(tmp_path) @@ -695,9 +721,11 @@ def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, di monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", original_write) with sqlite3.connect(active_index) as conn: - conn.execute("UPDATE raw_revision_heads SET session_id = 'changed' WHERE logical_source_key = 'k-r-present'") + conn.execute( + "UPDATE raw_revision_heads SET accepted_raw_id = 'r-gone' WHERE logical_source_key = 'k-r-present'" + ) - with pytest.raises(RawAuthorityRecoveryError, match="exact committed index seed state"): + with pytest.raises(RawAuthorityRecoveryError, match="unbacked index head"): resume_raw_authority_recovery( tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, @@ -761,6 +789,68 @@ def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, di ).fetchone() == (1,) +def test_index_prune_resume_accepts_source_backed_in_place_successor_head( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Receipt finalization admits a production-style replacement of a retained head row.""" + + initialize_active_archive_root(tmp_path) + monkeypatch.setattr(VERSION_INFO, "dirty", False) + active_index = _seed_index_seeds(tmp_path) + backup = _backup_authority(tmp_path, monkeypatch, tier="index") + plan = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.PRUNE_INDEX_SEEDS, backup_manifest=backup) + + from polylogue.maintenance import raw_authority_recovery + + original_write = raw_authority_recovery._write_durable_immutable + + def fail_final_receipt(root: Path, path: Path, payload: dict[str, object], *, digest_field: str) -> Path: + if digest_field == "receipt_sha256": + raise OSError("injected final receipt write failure") + return original_write(root, path, payload, digest_field=digest_field) + + monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", fail_final_receipt) + with pytest.raises(RawAuthorityRecoveryError, match="injected final receipt write failure"): + apply_raw_authority_recovery(plan) + monkeypatch.setattr(raw_authority_recovery, "_write_durable_immutable", original_write) + + from polylogue.archive.revision_replay import ApplicationDecision + from polylogue.storage.sqlite.archive_tiers.revision_application import ( + RevisionApplicationReceipt, + record_revision_application_sync, + ) + + with sqlite3.connect(active_index) as conn: + record_revision_application_sync( + conn, + RevisionApplicationReceipt( + raw_id="r-present", + session_id="s-successor", + logical_source_key="k-r-present", + source_revision="sr", + acquisition_generation=0, + decision=ApplicationDecision.SELECTED_BASELINE, + accepted_raw_id="r-present", + accepted_source_revision="sr", + accepted_content_hash=bytes.fromhex("04" * 32), + accepted_frontier_kind="byte", + accepted_frontier=1, + ), + decided_at_ms=2, + ) + + recovered = resume_raw_authority_recovery( + tmp_path, + RecoveryOperation.PRUNE_INDEX_SEEDS, + operation_id=plan.operation_id, + ) + assert recovered.status == "already_satisfied" + with sqlite3.connect(active_index) as conn: + assert conn.execute( + "SELECT session_id, accepted_content_hash FROM raw_revision_heads WHERE logical_source_key = 'k-r-present'" + ).fetchone() == ("s-successor", bytes.fromhex("04" * 32)) + + def test_census_reset_refuses_a_competing_source_continuity_intent( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/storage/test_durable_change_train.py b/tests/unit/storage/test_durable_change_train.py index acac029f07..1015fc7fb3 100644 --- a/tests/unit/storage/test_durable_change_train.py +++ b/tests/unit/storage/test_durable_change_train.py @@ -717,12 +717,55 @@ def test_startup_skips_unfinalized_raw_authority_receipt_and_recovers_other_inte evidence_ref="proof:other-recovery", ) - durable_change_train_module._recover_pending_source_continuity_intents(tmp_path) + assert durable_change_train_module._recover_pending_source_continuity_intents(tmp_path) == frozenset( + {ArchiveTier.SOURCE} + ) assert missing_pending.exists() assert not rolled_back_pending.exists() +def test_startup_defers_released_source_validation_for_an_unfinalized_reset_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A reset awaiting its receipt cannot be rejected by stale released-train evidence.""" + + source_db = tmp_path / "source.db" + _create_current_database(source_db) + with sqlite3.connect(source_db) as connection: + before = migration_runner.capture_durable_database_evidence(connection, ArchiveTier.SOURCE) + backup_manifest = tmp_path / "backup-manifest.json" + backup_manifest.write_text("{}\n", encoding="utf-8") + pending = write_source_continuity_pending_intent( + tmp_path, + mutation_receipt=tmp_path / "missing-raw-authority-receipt.json", + backup_manifest=backup_manifest, + pre_mutation_evidence=before, + operation_id="raw-authority-recovery", + evidence_ref="proof:raw-authority-recovery", + mutation_kind="raw_authority_recovery", + ) + manifest_path = tmp_path / ".maintenance-state" / "durable-change-trains" / "source-001.json" + manifest_path.parent.mkdir(parents=True) + manifest_path.touch() + released = cast( + DurableChangeTrain, + SimpleNamespace(state=DurableChangeTrainState.RELEASED, tier=ArchiveTier.SOURCE, target_version=1), + ) + + def fail_validation(*_args: object, **_kwargs: object) -> None: + pytest.fail("unfinalized reset receipt must defer released source-train validation") + + monkeypatch.setattr(durable_change_train_module, "DURABLE_MIGRATION_ADOPTION_FLOORS", {ArchiveTier.SOURCE: 0}) + monkeypatch.setattr(durable_change_train_module, "_fresh_durable_bootstrap_versions", lambda *_args: {}) + monkeypatch.setattr(durable_change_train_module, "load_durable_change_train_manifest", lambda _path: released) + monkeypatch.setattr(durable_change_train_module, "_released_train_manifests_by_target", fail_validation) + monkeypatch.setattr(durable_change_train_module, "_verify_released_train_live_tier", fail_validation) + + assert durable_change_train_module._reconcile_durable_change_train_startup_locked(tmp_path) == () + assert pending.exists() + + def test_startup_rejects_a_missing_liveness_receipt(tmp_path: Path) -> None: """Liveness pending evidence is corrupt when its prewritten receipt disappears.""" @@ -1523,7 +1566,7 @@ def fake_capture(*_args: object, **_kwargs: object) -> SimpleNamespace: monkeypatch.setattr( durable_change_train_module, "_recover_pending_source_continuity_intents", - lambda _root: None, + lambda _root: frozenset(), ) monkeypatch.setattr(durable_change_train_module, "_open_existing_tier", fake_open_tier) monkeypatch.setattr(durable_change_train_module, "load_durable_change_train_manifest", fake_load) @@ -1570,7 +1613,9 @@ def fake_open_tier(_path: Path) -> Iterator[sqlite3.Connection]: with sqlite3.connect(":memory:") as connection: yield connection - monkeypatch.setattr(durable_change_train_module, "_recover_pending_source_continuity_intents", lambda _root: None) + monkeypatch.setattr( + durable_change_train_module, "_recover_pending_source_continuity_intents", lambda _root: frozenset() + ) monkeypatch.setattr(durable_change_train_module, "_open_existing_tier", fake_open_tier) monkeypatch.setattr(durable_change_train_module, "load_durable_change_train_manifest", lambda _path: current) monkeypatch.setattr( @@ -1600,7 +1645,9 @@ def fake_open_tier(_path: Path) -> Iterator[sqlite3.Connection]: connection.execute("PRAGMA user_version = 28") yield connection - monkeypatch.setattr(durable_change_train_module, "_recover_pending_source_continuity_intents", lambda _root: None) + monkeypatch.setattr( + durable_change_train_module, "_recover_pending_source_continuity_intents", lambda _root: frozenset() + ) monkeypatch.setattr(durable_change_train_module, "_open_existing_tier", fake_open_tier) with pytest.raises(DurableChangeTrainError, match="lacks released train evidence"): From 2bc66ac5d3a567762472511b123e7a181ff0d078 Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 08:34:15 +0200 Subject: [PATCH 25/27] test(raw-authority): prove WAL ledger binding Assert that a WAL-visible census change alters the authorized recovery plan and is rejected before lease acquisition. --- tests/unit/maintenance/test_raw_authority_reset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/maintenance/test_raw_authority_reset.py b/tests/unit/maintenance/test_raw_authority_reset.py index a76838130f..a650ffa435 100644 --- a/tests/unit/maintenance/test_raw_authority_reset.py +++ b/tests/unit/maintenance/test_raw_authority_reset.py @@ -210,7 +210,10 @@ def test_census_reset_refuses_wal_visible_ledger_drift(tmp_path: Path, monkeypat assert source_db.read_bytes() == main_database_before assert source_db.with_name("source.db-wal").is_file() - with pytest.raises(RawAuthorityRecoveryError, match="census ledger snapshot"): + refreshed = inspect_raw_authority_recovery(tmp_path, RecoveryOperation.RESET_CENSUS, backup_manifest=backup) + assert refreshed.ledger_digest != plan.ledger_digest + assert refreshed.plan_digest != plan.plan_digest + with pytest.raises(RawAuthorityRecoveryError, match="stale before lease acquisition"): apply_raw_authority_recovery(plan) with sqlite3.connect(source_db) as conn: assert conn.execute("SELECT residual_json FROM raw_authority_censuses WHERE census_id = 'c1'").fetchone() == ( From 3800089847fd1b0a33ad49fbde4354238282215c Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 08:37:38 +0200 Subject: [PATCH 26/27] fix(raw-authority): preserve ledger digest type Keep recovery-plan deserialization aligned with the validated optional ledger digest type. --- polylogue/maintenance/raw_authority_recovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polylogue/maintenance/raw_authority_recovery.py b/polylogue/maintenance/raw_authority_recovery.py index 1078a5f9f3..8dec53c0e7 100644 --- a/polylogue/maintenance/raw_authority_recovery.py +++ b/polylogue/maintenance/raw_authority_recovery.py @@ -647,7 +647,7 @@ def from_dict(cls, payload: Mapping[str, object]) -> RawAuthorityRecoveryPlan: source_snapshot=str(payload["source_snapshot"]), active_generation=cast(dict[str, object], payload["active_generation"]), before_counts=_int_mapping(payload["before_counts"], field="before_counts"), - ledger_digest=cast(str | None, ledger_digest), + ledger_digest=ledger_digest, candidate_keys=candidates, post_target_proof=cast(dict[str, dict[str, object]] | None, post_target_proof), protected_digest=str(payload["protected_digest"]), From 8408b5bb1c83cf7b78a06677d6233c7b1def7a8e Mon Sep 17 00:00:00 2001 From: Sinity Date: Mon, 10 Aug 2026 09:09:13 +0200 Subject: [PATCH 27/27] style(raw-authority): format recovery helpers --- polylogue/storage/raw_authority.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/polylogue/storage/raw_authority.py b/polylogue/storage/raw_authority.py index d8830c85b3..899f4535c0 100644 --- a/polylogue/storage/raw_authority.py +++ b/polylogue/storage/raw_authority.py @@ -73,6 +73,7 @@ RAW_AUTHORITY_CENSUS_HEADER_RETENTION = 256 logger = get_logger(__name__) + def parser_census_logical_keys(logical_keys_json: object) -> tuple[str, ...] | None: """Validate and normalize the durable logical-key receipt payload. @@ -108,6 +109,7 @@ def parser_census_logical_keys(logical_keys_json: object) -> tuple[str, ...] | N "raw_authority_censuses", ) + @dataclass(frozen=True, slots=True) class RawAuthorityCensusResetCounts: """Counts for a reset of derived raw-authority census bookkeeping."""