From b454d16409b9873d8bfd860d8422afba506e04c7 Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Wed, 5 Aug 2026 02:58:09 +0300 Subject: [PATCH 1/2] feat(streaming): complete graph-first firehose controls --- AGENTS.md | 8 +- crates/omnigraph-api-types/src/lib.rs | 60 ++- crates/omnigraph-cli/src/cli.rs | 34 +- crates/omnigraph-cli/src/client.rs | 169 +++++++- crates/omnigraph-cli/src/main.rs | 61 ++- crates/omnigraph-cli/src/output.rs | 121 +++++- crates/omnigraph-cli/src/planes.rs | 55 ++- crates/omnigraph-cli/tests/cli_cluster.rs | 25 +- crates/omnigraph-cli/tests/system_local.rs | 256 ++++++++++++ crates/omnigraph-cluster/src/stream_block.rs | 321 +++++++++++++- crates/omnigraph-cluster/src/tests.rs | 65 ++- crates/omnigraph-cluster/src/types.rs | 92 +++- crates/omnigraph-server/src/handlers.rs | 141 +++++++ crates/omnigraph-server/src/lib.rs | 152 ++++++- crates/omnigraph-server/tests/auth_policy.rs | 23 + crates/omnigraph-server/tests/multi_graph.rs | 125 +++++- crates/omnigraph-server/tests/openapi.rs | 37 ++ crates/omnigraph/src/db/manifest.rs | 7 +- crates/omnigraph/src/db/manifest/stream.rs | 48 +++ .../omnigraph/src/db/manifest/token_store.rs | 150 +++++++ crates/omnigraph/src/db/mod.rs | 7 +- crates/omnigraph/src/db/omnigraph.rs | 9 +- .../src/db/omnigraph/stream_correction.rs | 251 ++++++++++- .../src/db/omnigraph/stream_driver.rs | 19 + .../src/db/omnigraph/stream_management.rs | 200 +++++++++ .../src/db/omnigraph/stream_retirement.rs | 316 +++++++++++--- .../omnigraph/src/db/omnigraph/table_ops.rs | 37 +- crates/omnigraph/tests/forbidden_apis.rs | 131 +++++- crates/omnigraph/tests/memwal_stream.rs | 356 ++++++++++++++-- docs/dev/canon.md | 39 +- docs/dev/invariants.md | 16 +- docs/dev/testing.md | 32 +- docs/dev/writing-path-state-of-affairs.md | 26 +- docs/releases/v0.10.0.md | 44 +- docs/rfcs/0026-memwal-streaming-ingest.md | 142 ++++--- docs/rfcs/0028-stable-schema-identity.md | 8 +- docs/user/cli/reference.md | 29 +- docs/user/clusters/config.md | 7 +- docs/user/clusters/index.md | 41 +- docs/user/operations/errors.md | 8 +- docs/user/operations/maintenance.md | 9 +- docs/user/operations/policy.md | 2 +- docs/user/operations/server.md | 45 +- docs/user/operations/upgrade.md | 8 +- openapi.json | 394 ++++++++++++++++++ 45 files changed, 3781 insertions(+), 345 deletions(-) create mode 100644 crates/omnigraph/src/db/omnigraph/stream_management.rs diff --git a/AGENTS.md b/AGENTS.md index 8695721f..7571de9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,14 +33,14 @@ OmniGraph is a typed property-graph engine built as a coordination layer over ma - **Multi-modal querying**: vector ANN (`nearest`), full-text (`search`/`fuzzy`/`match_text`/`bm25`), Reciprocal Rank Fusion (`rrf`), and graph traversal (`Expand`, anti-join `not { … }`) in one runtime. - **Branches and commits across the whole graph**: Git-style — every successful publish appends to a commit DAG; merges are three-way at the row level. - **Atomic per-query writes**: `mutate_as` and `load` accumulate insert/update batches into an in-memory `MutationStaging.pending` per touched table. Strict insert and upsert both route through the sealed exact-`id`, filter-bearing adapter; bare Lance Append is test-only. Their RFC-022 adapter resolves or rejects relevant recovery intents before base capture, captures `(native branch id, exact graph_head, schema identity)`, then rechecks recovery and authority under schema → branch → table gates. Mutation/Load keeps one keyed transaction per table and rejects more than 8,192 rows or 32 MiB before recovery arm; mutation update scans stream into the remaining table budget after pending-key shadowing, with blob sizes checked before payload reads. It then arms an identity-bearing recovery-v9 sidecar containing exact Lance transaction identities + pre-minted lineage, commits each table with zero transparent conflict retries, confirms the achieved effects, and publishes under the same token. Unrelated retryable pre-effect authority movement may fully reprepare without changing logical mode. A proven effect-free strict conflict returns `KeyConflict` only after a fresh exact-ID probe; no exact match triggers bounded full strict-mode reprepare, while an effect-free upsert conflict also fully reprepares. Any earlier effect or ambiguity returns `RecoveryRequired` with the sidecar retained. Strict read-set conflicts return `ReadSetChanged`. Deletes stage through the same path. D₂ at parse time remains the constructive (insert/update) XOR destructive (delete) boundary. -- **RFC-026 private streaming core**: Phase A's main-only adapter enrolls one exact-`id` table into one empty unsharded Lance MemWAL; internal schema v12 moved that hidden path to recovery-v14 `StreamEnrollmentV2`, which selects exact enrollment/binding ledger receipts with the physical binding and `OPEN` lifecycle. Phase B1 adds a root-singleflight worker for one no-roll generation (8,192 rows / 32 MiB of logical dense-slice Arrow data), watcher success plus the same writer's post-durability `check_fenced()` before clean acknowledgement, conservative replay/fold-only recovery, and the bounded fold mechanics. Cheap raw bounds and exact post-tombstone validation run before recovery; resident-producing served put ownership follows bounded preprocessing/inflight → root MemWAL opportunity shared → profile shared → table admission → same-key queue → worker mode; cold replay installs exact fold-only accounting; exclusive admission spans claim, seal, drain proof, both fold effects, and publication. Admission, replay, and fold all charge `ArrayData::get_slice_memory_size`; fold densifies retained arrays. Backing-buffer capacity and physical allocation are not the limit, while isolated fold RSS remains a 384-MiB remeasurement tripwire. The 8,192-row high-entropy near-cap closure cell remains green. Internal schema v9 added the private B2 compare-and-chain core: canonical payload and token digests, grammar-impossible trusted hidden row metadata (`__omnigraph_stream_v1$`), same-generation token overlays, and a manifest-selected graph-global `_stream_tokens.lance` authority. Exact base/token lookups and recovery validation materialize at most the requested rows plus one and cap both Arrow batch bytes and cumulative retained bytes. Admission recaptures lifecycle/binding/HEAD authority only after acquiring shared admission and the same-key queue, so a stale provisional capture cannot authorize a WAL put. Under lifecycle-v3, every cold opener and ordinary fold first publishes an exact recovery-v14 writer claim whose immutable attempt/terminal receipts authenticate the bounded WAL suffix and full current-generation LWW projection. `StreamFoldV2` and `StreamDrainFold` own exact pre-minted base/token transactions, bind the selected current claim and authenticated tail, and publish the two pointers, one lifecycle successor, and graph lineage together. Quiescence is restartable `OPEN → DRAINING → SEALED`: it supports never-written empty lanes, folds non-empty lanes, reuses an already-flushed exact cut after restart, and emits one immutable terminal management receipt. V13's recovery-v15 `StreamResume` privately consumes an exact sealed proof or guarded drain state, owns the higher-epoch claim plus terminal claim/management receipts, and alone publishes the next `OPEN` row. Receipt-first replay is idempotent, and named-branch or stale-binding authority fails before the claim. V14's recovery-v16 `StreamSealedEnsureIndices` layers exact prior/next SEALED rows over recovery-v8's CreateIndex plan; under `stream_manage` and the retained checked runtime it publishes every productive index pointer, both HEAD witnesses, recomputed empty proof, and lifecycle revision in one CAS. V15's recovery-v17 `StreamSealedOptimize` separately owns Optimize's internally committing maintenance plan and exact achieved HEADs, then publishes productive table pointers with recomputed `SEALED` proofs in one CAS. Neither checked maintenance path writes a token row, advances a management-receipt chain, or accepts a caller operation ID; true no-work remains effect-free. V16's recovery-v18 `StreamRebind` consumes one exact `SEALED` proof, creates a fresh enrollment and empty shard namespace, appends immutable binding and fence-only claim receipts, and publishes a new exact `SEALED` proof without admitting a writer or put. A separate recovery-v15 resume is required to open the fresh scope. This physical rebind keeps the accepted schema unchanged. Productive SchemaApply remains refused on any enrolled graph in EXP; schema changes use a checked sealed/retired export, initialize a fresh graph with the desired schema, and load there, never over the enrolled source. V17's recovery-v19 `StreamAuthorityRetirement` owns the exact immutable retirement-receipt transaction and sole lineage-neutral `DISABLED → RETIRED` publication; it never moves graph or branch heads and its selected receipt/profile chain replaces `RecoveryAudit`. V18's recovery-v20 `StreamCorrection` owns one exact blocked generation cut, a pre-minted base effect, and one combined token-successor plus correction/management-receipt effect; recovery may recreate only the missing token participant after the exact base effect and publishes the two pointers, `DRAINING` successor, lineage, and optional PRESENT attribution together. V19's recovery-v21 `DeadLetterFold` owns one conditionally created canonical object plus exact base and token effects, accepts mixed and all-diverted outcomes, and publishes versioned fold attribution with current token-schema-v3 `DEAD_LETTERED` evidence. Recovery-v21 also carries the three-disposition retirement owner; recovery-v19 retains its historical two-disposition meaning. Ambient EnsureIndices and Optimize remain refused for enrolled tables. Historical recovery-v10 enrollment, recovery-v12 lifecycle-v2 fold, and older resume/maintenance/rebind scaffolds keep their old wire meanings and are refused rather than synthesized. Post-invocation ambiguity is `AckUnknown`; unresolved recovery blocks progress. The topology remains main-only, unsharded, one resident writer and one live writer process, with no fresh reads or generation GC. The private **B2a unbounded retain-all profile is active**: OmniGraph imposes no retained-byte, object-count, file-count, or history quota and never deletes a canonical durable `_mem_wal` object. Lance may clean only its losing `.binpb.tmp.` atomic-CAS staging. Complete and partial unreferenced generation residue stays non-authoritative and untouched through recovery/reopen; provider exhaustion is loud. The core remains crate-private behind feature-gated, doc-hidden test seams. The graph-scoped `stream_ingest` / `stream_manage` Cedar vocabulary is registered, and embedded `Omnigraph::stream_status` exposes a durable manifest-only projection. V11's protocol-v2 checked cluster authority remains unchanged: `ENABLED` carries its active fold delegation, `DISABLING` carries a drain-only continuation plan, and `RETIRED` is receipt/cut-bound and fail-closed. Recovery-v13 remains exactly `StreamProfileChange`. Profile apply enforces `stream_manage` under both the currently applied and desired graph policies through the cluster-state CAS. Enabled-profile Mutation/Load/delete require the exact checked served runtime, while BranchMerge remains closed under both `ENABLED` and `DISABLING` even with that runtime. The narrow cluster-only `stream retire-for-rebuild plan|confirm` command is the sole active retirement surface and permanently freezes a verified `WITHDRAWN | DEAD_LETTERED` cut for export/rebuild. Checked offline disable is the sole supported production quiescence owner. F7a activates one graph-native mixed node/edge served-ingest bridge at `POST /graphs/{graph_id}/stream/ingest` plus remote client/CLI/OpenAPI parity. A strong graph-authority ETag gates body ownership; the engine resolves declarations, lazily prepares private lanes, and returns only ordered graph-logical results. F7b exposes F6b6's checked read-only cut at `GET /graphs/{graph_id}/stream/status` and `stream status` through a graph-logical projection that omits every table/dataset/lane/binding/shard/generation/Lance/recovery identity. Public lane enrollment, general resume/abort, rebind, and lifecycle/maintenance transports remain inactive; exact-terminal served export remains the F6b5 exception. The stopped/offline cluster-only `stream block show|correct` and `stream dead-letter list|export` surfaces are the narrow active inspection/correction exceptions; no hidden seam is a supported operator escape hatch. -- **RFC-026 F7a lifecycle boundary**: graph firehose ingress can lazily prepare an absent declaration or continue an existing `OPEN` lane. It never implicitly resumes a `SEALED` lane; that state returns graph-level `stream_authority_changed` until a public checked resume exists, or the operator takes the documented retire/export/rebuild exit. +- **RFC-026 private streaming core**: Phase A's main-only adapter enrolls one exact-`id` table into one empty unsharded Lance MemWAL; internal schema v12 moved that hidden path to recovery-v14 `StreamEnrollmentV2`, which selects exact enrollment/binding ledger receipts with the physical binding and `OPEN` lifecycle. Phase B1 adds a root-singleflight worker for one no-roll generation (8,192 rows / 32 MiB of logical dense-slice Arrow data), watcher success plus the same writer's post-durability `check_fenced()` before clean acknowledgement, conservative replay/fold-only recovery, and the bounded fold mechanics. Cheap raw bounds and exact post-tombstone validation run before recovery; resident-producing served put ownership follows bounded preprocessing/inflight → root MemWAL opportunity shared → profile shared → table admission → same-key queue → worker mode; cold replay installs exact fold-only accounting; exclusive admission spans claim, seal, drain proof, both fold effects, and publication. Admission, replay, and fold all charge `ArrayData::get_slice_memory_size`; fold densifies retained arrays. Backing-buffer capacity and physical allocation are not the limit, while isolated fold RSS remains a 384-MiB remeasurement tripwire. The 8,192-row high-entropy near-cap closure cell remains green. Internal schema v9 added the private B2 compare-and-chain core: canonical payload and token digests, grammar-impossible trusted hidden row metadata (`__omnigraph_stream_v1$`), same-generation token overlays, and a manifest-selected graph-global `_stream_tokens.lance` authority. Exact base/token lookups and recovery validation materialize at most the requested rows plus one and cap both Arrow batch bytes and cumulative retained bytes. Admission recaptures lifecycle/binding/HEAD authority only after acquiring shared admission and the same-key queue, so a stale provisional capture cannot authorize a WAL put. Under lifecycle-v3, every cold opener and ordinary fold first publishes an exact recovery-v14 writer claim whose immutable attempt/terminal receipts authenticate the bounded WAL suffix and full current-generation LWW projection. `StreamFoldV2` and `StreamDrainFold` own exact pre-minted base/token transactions, bind the selected current claim and authenticated tail, and publish the two pointers, one lifecycle successor, and graph lineage together. Quiescence is restartable `OPEN → DRAINING → SEALED`: it supports never-written empty lanes, folds non-empty lanes, reuses an already-flushed exact cut after restart, and emits one immutable terminal management receipt. V13's recovery-v15 `StreamResume` privately consumes an exact sealed proof or guarded drain state, owns the higher-epoch claim plus terminal claim/management receipts, and alone publishes the next `OPEN` row. Receipt-first replay is idempotent, and named-branch or stale-binding authority fails before the claim. V14's recovery-v16 `StreamSealedEnsureIndices` layers exact prior/next SEALED rows over recovery-v8's CreateIndex plan; under `stream_manage` and the retained checked runtime it publishes every productive index pointer, both HEAD witnesses, recomputed empty proof, and lifecycle revision in one CAS. V15's recovery-v17 `StreamSealedOptimize` separately owns Optimize's internally committing maintenance plan and exact achieved HEADs, then publishes productive table pointers with recomputed `SEALED` proofs in one CAS. Neither checked maintenance path writes a token row, advances a management-receipt chain, or accepts a caller operation ID; true no-work remains effect-free. V16's recovery-v18 `StreamRebind` consumes one exact `SEALED` proof, creates a fresh enrollment and empty shard namespace, appends immutable binding and fence-only claim receipts, and publishes a new exact `SEALED` proof without admitting a writer or put. A separate recovery-v15 resume is required to open the fresh scope. This physical rebind keeps the accepted schema unchanged. Productive SchemaApply remains refused on any enrolled graph in EXP; schema changes use a checked sealed/retired export, initialize a fresh graph with the desired schema, and load there, never over the enrolled source. V17's recovery-v19 `StreamAuthorityRetirement` owns the exact immutable retirement-receipt transaction and sole lineage-neutral `DISABLED → RETIRED` publication; it never moves graph or branch heads and its selected receipt/profile chain replaces `RecoveryAudit`. V18's recovery-v20 `StreamCorrection` owns one exact blocked generation cut, a pre-minted base effect, and one combined token-successor plus correction/management-receipt effect; recovery may recreate only the missing token participant after the exact base effect and publishes the two pointers, `DRAINING` successor, lineage, and optional PRESENT attribution together. V19's recovery-v21 `DeadLetterFold` owns one conditionally created canonical object plus exact base and token effects, accepts mixed and all-diverted outcomes, and publishes versioned fold attribution with current token-schema-v3 `DEAD_LETTERED` evidence. Recovery-v21 also carries the three-disposition retirement owner; recovery-v19 retains its historical two-disposition meaning. Ambient EnsureIndices and Optimize remain refused for enrolled tables. Historical recovery-v10 enrollment, recovery-v12 lifecycle-v2 fold, and older resume/maintenance/rebind scaffolds keep their old wire meanings and are refused rather than synthesized. Post-invocation ambiguity is `AckUnknown`; unresolved recovery blocks progress. The topology remains main-only, unsharded, one resident writer and one live writer process, with no fresh reads or generation GC. The private **B2a unbounded retain-all profile is active**: OmniGraph imposes no retained-byte, object-count, file-count, or history quota and never deletes a canonical durable `_mem_wal` object. Lance may clean only its losing `.binpb.tmp.` atomic-CAS staging. Complete and partial unreferenced generation residue stays non-authoritative and untouched through recovery/reopen; provider exhaustion is loud. The core remains crate-private behind feature-gated, doc-hidden test seams. The graph-scoped `stream_ingest` / `stream_manage` Cedar vocabulary is registered, and embedded `Omnigraph::stream_status` exposes a durable manifest-only projection. V11's protocol-v2 checked cluster authority remains unchanged: `ENABLED` carries its active fold delegation, `DISABLING` carries a drain-only continuation plan, and `RETIRED` is receipt/cut-bound and fail-closed. Recovery-v13 remains exactly `StreamProfileChange`. Profile apply enforces `stream_manage` under both the currently applied and desired graph policies through the cluster-state CAS. Enabled-profile Mutation/Load/delete require the exact checked served runtime, while BranchMerge remains closed under both `ENABLED` and `DISABLING` even with that runtime. The narrow cluster-only `stream retire-for-rebuild plan|confirm` command is the sole active retirement surface and permanently freezes a verified `WITHDRAWN | DEAD_LETTERED` cut for export/rebuild. Checked offline disable is the sole supported production quiescence owner. F7a activates one graph-native mixed node/edge served-ingest bridge at `POST /graphs/{graph_id}/stream/ingest` plus remote client/CLI/OpenAPI parity. A strong graph-authority ETag gates body ownership; the engine resolves declarations, lazily prepares private lanes, and returns only ordered graph-logical results. F7b exposes F6b6's checked read-only cut at `GET /graphs/{graph_id}/stream/status` and `stream status` through a graph-logical projection that omits every table/dataset/lane/binding/shard/generation/Lance/recovery identity. F7c exposes only selector-free graph-wide resume plus SEALED EnsureIndices/Optimize, composing recovery-v15/v16/v17 without a new coordinator or format. Public per-declaration enrollment/resume/abort, rebind, and general lifecycle control remain inactive; exact-terminal served export remains the F6b5 exception. The stopped/offline cluster-only `stream block show|correct` and `stream dead-letter list|export` surfaces are the narrow active inspection/correction exceptions; no hidden seam is a supported operator escape hatch. +- **RFC-026 graph-wide lifecycle boundary**: graph firehose ingress can lazily prepare an absent declaration or continue an existing `OPEN` lane. It never implicitly resumes `SEALED`; the explicit served graph-wide resume preflights every enrolled declaration, refuses any drain/strict block, skips `OPEN`, and composes recovery-v15 for the deterministic `SEALED` remainder. It accepts no declaration/table selector. Per-declaration resume/abort and rebind remain private. - **RFC-026 retired-export provenance**: the immutable retirement receipt binds the whole frozen graph cut. Every retired JSONL export pairs that root receipt with a selected `branch_member` witness containing the canonical branch name, exact Lance branch identifier, graph head, manifest version, `table_witness_digest`, and a recomputable `branch_member_digest`. The export's `source_schema_ir_hash`, exact `ordered_branch_member_digests`, and `selected_member_index` let load prove membership and recompute the receipt's `export_cut_digest`. The source schema hash commits the retired source cut; it is not required to equal the fresh target graph identity, whose compatibility remains ordinary loader validation. Load imports no live stream authority. - **RFC-026 B2 preprocessing bound**: before blob materialization or canonical encoding, the private adapter reserves a 128-MiB worst-case envelope (original 32-MiB Arrow row + possible 32-MiB replacement + 64-MiB canonical payload) and an inflight slot. The private profile admits exactly two envelopes (256 MiB root-wide), preserving the two-caller stale-authority race without unbounded preprocessing. The slot transfers into queued/worker ownership; scratch releases after digest derivation. Pressure fails effect-free as typed `stream_b2_preprocessing_bytes`; this is process-memory admission, not retained-storage GC or quota. - **HTTP server**: Axum + utoipa OpenAPI, bearer auth (SHA-256 hashed, optional AWS Secrets Manager). Cedar policy enforcement is engine-wide — every `_as` writer calls `Omnigraph::enforce(action, scope, actor)`, so HTTP, CLI, and embedded SDK consumers all hit the same gate. **Cluster-only boot** (RFC-011): the server always boots from a cluster directory (`--cluster `, RFC-005) and serves N graphs (N ≥ 1) under multi-graph routes (`/graphs/{graph_id}/...` + read-only `GET /graphs` enumeration); there are no single-graph flat routes and no positional-URI boot. Per-graph + server-level Cedar policies. Runtime add/remove (`POST /graphs`, `DELETE /graphs/{id}`) is not exposed — operators run `cluster apply` and restart. - **CLI** with two-surface config (RFC-007/008): the team-owned cluster directory (`cluster.yaml`) plus the per-operator `~/.omnigraph/config.yaml` (servers, clusters, credentials, actor, profiles, aliases, defaults). Graphs are addressed via `--store`/`--server`/`--cluster`/`--profile`/operator defaults (RFC-011). Multi-format output (json/jsonl/csv/kv/table). -- **RFC-026 hidden F4/F5/F6 status**: F4 owns the complete caller-shaped authorized JSON/NDJSON and bodyless lazy-enrollment prepare path. Format-neutral F5a adds one weakly root-scoped automatic supervisor for `OPEN` lanes. Detached ownership schedules a coalesced timer wake immediately after physical put invocation, including caller cancellation or eventual `AckUnknown`; passive readiness discards no-effect wakes, and capacity pressure makes the same entry urgent. Cold start derives backlog from manifest plus authenticated MemWAL authority, while finite rounds visit nodes before edges with a carried round-robin cursor inside each immutable-identity cohort. F5b0 extends that exact-`ENABLED`, checked-runtime owner to restart and continue unblocked `DRAINING(goal = SEALED)` lanes through the existing recovery-v14 quiesce adapter. The same format-neutral cut makes checked offline `cluster apply` publish `DISABLING` before work, derive its finite lane set from that manifest cut, visit nodes then edges in immutable-identity order one lane at a time, continue `OPEN` and goal-`SEALED` drains, and recovery-v14-adopt an existing `OPEN_AFTER_FOLD` drain without minting a second occurrence. A selected `DataBlock` parks apply loudly at the durable `DISABLING` revision; stopped/offline correction followed by the same apply retry resumes the stored plan. F5b adds the v19/recovery-v21 terminal path: deterministic conflict partitioning can publish visible winners and one bounded dead-letter object, including a marker-only base transaction when every candidate is diverted; exact retries return the current terminal result, while a fresh ordinary successor naming that terminal predecessor restores `PRESENT`. Stopped/offline `cluster stream dead-letter list|export` inspects only manifest-selected current authority, and retirement now accepts `WITHDRAWN | DEAD_LETTERED`. The resident and offline owners retain their non-overlap boundary. After listener bind the cluster server starts every selected graph supervisor; graceful shutdown fences the root MemWAL opportunity exclusively and then the profile gate exclusively, drops both before joining the drivers concurrently under one bounded deadline and retains any timed-out live owner. F6a adds one typed failpoints-only process-local advisory driver snapshot and one hidden in-process composed acceptance spanning prepare, NDJSON, automatic mixed terminal fold, stopped/offline list/export, corrected successor, driver restart, clean shutdown ownership, and offline disable. F6b1 adds a distinct lower/engine checked served-export authority and one doc-hidden, move-only `StreamExportCut`. A managed exact `DISABLED | RETIRED` row binds directly; when the streaming row was legitimately removed, exact graph/state evidence binds only if the engine proves `RETIRED` or enrolled `DISABLED`. Retirement confirmation CAS-converges a managed row to `RETIRED` and its exact revision, while refresh preserves declaration identity and treats `RETIRED` as satisfying `streaming: false`. Ambient enrolled `DISABLED` export refuses. The receipt-verified ambient `RETIRED` direct-export bridge remains, while F6b5 routes served export through the checked cut. Checked capture non-waitingly owns the sole root slot, closes profile/admission/schema/branch/token/table gates, prevalidates filters and terminal authority, and freezes the accepted catalog, selected snapshot's exact Lance table versions, and retired provenance. It then drops all writer gates while the cut retains its checked process authority and root slot through consuming output, so a later writer cannot retarget the bytes. Branch create/create-from/delete, cleanup, schema apply, and supported whole-root deletion acquire the same slot non-waitingly, excluding path/version reuse for the cut's lifetime. The temporary `RETIRED` bridge also retains that slot through output. Refusal is before output; a storage or writer failure after output starts remains that stream error. Pending triggers are not a durable backlog, stopped driver state is not offline authority, and public durable `StreamStatus` remains manifest-only. F6b1 changed no format or recovery grammar; F6b5 subsequently activated the existing HTTP/remote-client/CLI/OpenAPI export route. F7a now activates only the graph-native served row route and remote command over the same hidden runtime, with no new coordinator, format, or recovery grammar. F7b exposes F6b6's checked cut through a graph-redacted read-only HTTP/OpenAPI route and remote CLI while the ambient manifest-only status remains unchanged; lifecycle/maintenance transports remain inactive; F6b2 now has green SIGTERM/shared shutdown, sequential OS-process recovery, frozen-round node/edge fairness, rebind/re-enable/reopen/resume, combined maintenance, fresh-target import, and legacy writer-refusal cells and is implemented. F6b3 adds exact manifest-selected token-index coverage probes plus a fixed-cardinality cost harness: zero-lane profile cycles grow immutable token-ledger receipt history while graph-manifest history also advances, then one enrolled conflicting key produces one logical terminal entry at every depth. The fast local 1/8-cycle cell and ignored local/configured-RustFS 1/8/32/128 sweeps record per-sample coverage, page bytes, and cumulative advisory whole-process RSS. Fresh-handle hit/miss plus the first terminal page and same-handle warm hit/miss plus repeat terminal pages report token-read counts, total table-store read bytes, manifest reads/bytes, adapter-operation counts, and per-sample warm/repeat p50 plus max-of-eight; graph open precedes those windows, so this is not a cold-provider-cache claim. The instrument does not query receipt keys and fails if a measured operation writes authority, touches MemWAL/base rows, lists prefixes, or fetches the dead-letter payload object. This remains the historical F6b3 uncovered-tail evidence. F6b7 adds a paired failpoints-only content-identical selected-index cut, current-token and profile-receipt lookup measurements, and maintenance-cost accounting. Its configured-RustFS result is a bounded NO-GO only for the uncompacted profile-cycle fixture, so it schedules no standalone production reconciler. F6b4 closes production dead-letter envelope/peak-RSS evidence. F6b5 adds pre-header checked cut capture, incremental exact-version scans, strict 64-KiB chunks, a two-chunk queue, complete per-response/process queue reservation, body-plus-producer lease ownership, terminal-frame cut retention, backpressure, and disconnect-safe release. F6b6 implements one engine-internal checked read-only operational cut: exact profile-specific authority; one writer-unfenced immutable token/base, bounded-sample, coverage, and lifecycle-ledger preflight; then short root/profile/lane fences for mutable physical/recovery witnesses; an explicitly advisory driver projection; typed movement/busy refusal; a complete pending-sidecar inventory inside a hard advisory envelope of 256 matching direct `.json` sidecars, 256 irrelevant direct-or-nested objects encountered below the prefix, 4 MiB of cumulative input-anchored URI bytes across all encountered objects, 32 MiB per sidecar body, and 32 MiB of cumulative bodies; and honest unavailable states for exactly recovery-owned physical movement, cold replay, flushed LWW accounting, and exact oldest-uncovered age. Exceeding any envelope bound is a typed resource refusal, never partial status. It neither heals recovery nor opens/claims/seals/folds a writer; the public status remains manifest-only. F6b8 separately closes resume-to-driver ownership without a format change: the root producer permit transfers through detached installation, an urgent trigger is armed before release, and exact empty-owner housekeeping precedes the unchanged node-before-edge round. No active path produces reserved `AuthorityBlock`, so its repair remains fail-closed. +- **RFC-026 hidden F4/F5/F6 status**: F4 owns the complete caller-shaped authorized JSON/NDJSON and bodyless lazy-enrollment prepare path. Format-neutral F5a adds one weakly root-scoped automatic supervisor for `OPEN` lanes. Detached ownership schedules a coalesced timer wake immediately after physical put invocation, including caller cancellation or eventual `AckUnknown`; passive readiness discards no-effect wakes, and capacity pressure makes the same entry urgent. Cold start derives backlog from manifest plus authenticated MemWAL authority, while finite rounds visit nodes before edges with a carried round-robin cursor inside each immutable-identity cohort. F5b0 extends that exact-`ENABLED`, checked-runtime owner to restart and continue unblocked `DRAINING(goal = SEALED)` lanes through the existing recovery-v14 quiesce adapter. The same format-neutral cut makes checked offline `cluster apply` publish `DISABLING` before work, derive its finite lane set from that manifest cut, visit nodes then edges in immutable-identity order one lane at a time, continue `OPEN` and goal-`SEALED` drains, and recovery-v14-adopt an existing `OPEN_AFTER_FOLD` drain without minting a second occurrence. A selected `DataBlock` parks apply loudly at the durable `DISABLING` revision; stopped/offline correction followed by the same apply retry resumes the stored plan. F5b adds the v19/recovery-v21 terminal path: deterministic conflict partitioning can publish visible winners and one bounded dead-letter object, including a marker-only base transaction when every candidate is diverted; exact retries return the current terminal result, while a fresh ordinary successor naming that terminal predecessor restores `PRESENT`. Stopped/offline `cluster stream dead-letter list|export` inspects only manifest-selected current authority, and retirement now accepts `WITHDRAWN | DEAD_LETTERED`. The resident and offline owners retain their non-overlap boundary. After listener bind the cluster server starts every selected graph supervisor; graceful shutdown fences the root MemWAL opportunity exclusively and then the profile gate exclusively, drops both before joining the drivers concurrently under one bounded deadline and retains any timed-out live owner. F6a adds one typed failpoints-only process-local advisory driver snapshot and one hidden in-process composed acceptance spanning prepare, NDJSON, automatic mixed terminal fold, stopped/offline list/export, corrected successor, driver restart, clean shutdown ownership, and offline disable. F6b1 adds a distinct lower/engine checked served-export authority and one doc-hidden, move-only `StreamExportCut`. A managed exact `DISABLED | RETIRED` row binds directly; when the streaming row was legitimately removed, exact graph/state evidence binds only if the engine proves `RETIRED` or enrolled `DISABLED`. Retirement confirmation CAS-converges a managed row to `RETIRED` and its exact revision, while refresh preserves declaration identity and treats `RETIRED` as satisfying `streaming: false`. Ambient enrolled `DISABLED` export refuses. The receipt-verified ambient `RETIRED` direct-export bridge remains, while F6b5 routes served export through the checked cut. Checked capture non-waitingly owns the sole root slot, closes profile/admission/schema/branch/token/table gates, prevalidates filters and terminal authority, and freezes the accepted catalog, selected snapshot's exact Lance table versions, and retired provenance. It then drops all writer gates while the cut retains its checked process authority and root slot through consuming output, so a later writer cannot retarget the bytes. Branch create/create-from/delete, cleanup, schema apply, and supported whole-root deletion acquire the same slot non-waitingly, excluding path/version reuse for the cut's lifetime. The temporary `RETIRED` bridge also retains that slot through output. Refusal is before output; a storage or writer failure after output starts remains that stream error. Pending triggers are not a durable backlog, stopped driver state is not offline authority, and public durable `StreamStatus` remains manifest-only. F6b1 changed no format or recovery grammar; F6b5 subsequently activated the existing HTTP/remote-client/CLI/OpenAPI export route. F7a now activates only the graph-native served row route and remote command over the same hidden runtime, with no new coordinator, format, or recovery grammar. F7b exposes F6b6's checked cut through a graph-redacted read-only HTTP/OpenAPI route and remote CLI while the ambient manifest-only status remains unchanged; F7c exposes graph-wide resume and SEALED maintenance only, with no physical selector; F6b2 now has green SIGTERM/shared shutdown, sequential OS-process recovery, frozen-round node/edge fairness, rebind/re-enable/reopen/resume, combined maintenance, fresh-target import, and legacy writer-refusal cells and is implemented. F6b3 adds exact manifest-selected token-index coverage probes plus a fixed-cardinality cost harness: zero-lane profile cycles grow immutable token-ledger receipt history while graph-manifest history also advances, then one enrolled conflicting key produces one logical terminal entry at every depth. The fast local 1/8-cycle cell and ignored local/configured-RustFS 1/8/32/128 sweeps record per-sample coverage, page bytes, and cumulative advisory whole-process RSS. Fresh-handle hit/miss plus the first terminal page and same-handle warm hit/miss plus repeat terminal pages report token-read counts, total table-store read bytes, manifest reads/bytes, adapter-operation counts, and per-sample warm/repeat p50 plus max-of-eight; graph open precedes those windows, so this is not a cold-provider-cache claim. The instrument does not query receipt keys and fails if a measured operation writes authority, touches MemWAL/base rows, lists prefixes, or fetches the dead-letter payload object. This remains the historical F6b3 uncovered-tail evidence. F6b7 adds a paired failpoints-only content-identical selected-index cut, current-token and profile-receipt lookup measurements, and maintenance-cost accounting. Its configured-RustFS result is a bounded NO-GO only for the uncompacted profile-cycle fixture, so it schedules no standalone production reconciler. F6b4 closes production dead-letter envelope/peak-RSS evidence. F6b5 adds pre-header checked cut capture, incremental exact-version scans, strict 64-KiB chunks, a two-chunk queue, complete per-response/process queue reservation, body-plus-producer lease ownership, terminal-frame cut retention, backpressure, and disconnect-safe release. F6b6 implements one engine-internal checked read-only operational cut: exact profile-specific authority; one writer-unfenced immutable token/base, bounded-sample, coverage, and lifecycle-ledger preflight; then short root/profile/lane fences for mutable physical/recovery witnesses; an explicitly advisory driver projection; typed movement/busy refusal; a complete pending-sidecar inventory inside a hard advisory envelope of 256 matching direct `.json` sidecars, 256 irrelevant direct-or-nested objects encountered below the prefix, 4 MiB of cumulative input-anchored URI bytes across all encountered objects, 32 MiB per sidecar body, and 32 MiB of cumulative bodies; and honest unavailable states for exactly recovery-owned physical movement, cold replay, flushed LWW accounting, and exact oldest-uncovered age. Exceeding any envelope bound is a typed resource refusal, never partial status. It neither heals recovery nor opens/claims/seals/folds a writer; the public status remains manifest-only. F6b8 separately closes resume-to-driver ownership without a format change: the root producer permit transfers through detached installation, an urgent trigger is armed before release, and exact empty-owner housekeeping precedes the unchanged node-before-edge round. No active path produces reserved `AuthorityBlock`, so its repair remains fail-closed. - **RFC-026 F6b2/F6b8 implemented acceptance scope**: this no-format slice has green Unix `SIGTERM` through the shared graceful-shutdown path, sequential OS-process recovery, a frozen-round node/edge non-starvation proof, terminal-disable physical rebind → re-enable → reopen → explicit resume → exactly-once ingest/fold, combined `quiesce → EnsureIndices → Optimize → resume`, checked-cut fresh-target import, and legacy Mutation/Load/delete, `load_file`, and corresponding `_as` refusal under `ENABLED` and interrupted `DISABLING`. Resident-producing served puts acquire bounded preprocessing/inflight → root MemWAL opportunity shared → profile shared → table admission; the driver holds root opportunity exclusive across one frozen round and then takes profile/admission per candidate. Producer and round permits retain the `MemWalWorkerRegistry` `Arc`, preventing weak-root fence ABA. Shutdown fences root exclusive and then profile exclusive, drops both, and joins the driver. F6b8 closes the empty-resume handoff: resume transfers its non-clone root producer permit into detached installation and retained retirement, arms an urgent trigger before release, and the driver retires only exact empty owners under lane-exclusive authority before preserving the ordinary node-before-edge round. Driver-first, caller-cancelled resume-first, cross-lane slot reuse, and clean-shutdown cells are green; the broader post-claim install/retirement-failure matrix remains later F6 work. Productive SchemaApply remains refused on enrolled graphs; EXP schema changes use checked export/init/load into a fresh graph, while physical rebind preserves accepted schema. F6b7 now owns the paired uncovered/reconciled token decision instrument; lifecycle/maintenance and direct-SDK status parity remain later work; F7a graph-native served row ingest, F6b5 exact-terminal served export, and F7b graph-redacted HTTP/OpenAPI/remote-CLI status are active over F6b6's checked operational core. - **RFC-026 F6b3/F6b7 token-index evidence**: failpoints-only instruments compare manifest-selected current-token and profile-receipt lookups before and after a @@ -283,7 +283,7 @@ omnigraph policy explain --cluster ./company-brain --graph knowledge --actor act | Cleanup (`cleanup_old_versions`) | ✅ | `omnigraph cleanup` derives requested `--keep` / `--older-than` cutoffs from each table's available versions; Lance refs plus OmniGraph's live-lazy-branch and recovery floors may retain additional versions. It fails closed on unopenable pins, recovery intent, or uncovered main-table HEAD drift | | BTREE / inverted (FTS) / vector indexes | ✅ | `@index`/`@key` declares intent; the physical index is derived state that never fails a logical op. Built per column through one chokepoint (`build_indices_on_dataset_for_catalog`, type-dispatched by `node_prop_index_kind`: enum + orderable scalar → BTREE, free-text String → FTS, Vector → vector); idempotent; lazy across branches. **Schema apply and mutation/load build no indexes inline**: they publish only logical data/schema effects, leaving physical intent pending. `ensure_indices` first runs the roll-forward-only recovery barrier before base capture or planning, then materializes every declared-but-missing artifact for one table through one staged mixed CreateIndex transaction under its identity-bearing recovery-v9 authority/lineage/delta envelope; it continues to report untrainable Vector columns as pending. `Armed` is rollback-only, `EffectsConfirmed` rolls forward only while captured authority holds, and first-touch refs carry exact identity. The private v14 checked-runtime main-only seam instead uses recovery-v16 when any productive table is enrolled: all such lanes must be exactly `SEALED`, and one CAS publishes every table pointer with its recomputed proof and next lifecycle revision. V15 adds the separate recovery-v17 checked `SEALED` Optimize bridge for bounded internally committing compaction/index-maintenance effects. Neither path creates a token receipt or caller operation occurrence; ambient EnsureIndices and Optimize remain refused for enrolled tables. | | Strict insert / upsert ingestion | ✅ transaction conflict filters + uncommitted fragment staging | Internal schema v6 introduced the explicit logical mode and v9 preserves it. General strict insert and upsert use the sealed exact-`id`, forced-v2 MergeInsert adapter; strict insert exact-probes its pinned parent before minting `omnigraph.insert_absence=v1`, while an all-new upsert may mint the same optional certificate only from its completed effect statistics. Mutation/Load remains one transaction per table, capped before arm at 8,192 rows / 32 MiB. BranchMerge's proven all-new route accepts only a complete certificate chain plus final source/target native-incarnation checks; it stages bounded fragments with `InsertBuilder`, commits them as exact-`id` filtered `Update` transactions, and performs zero target preflights, target merge joins, or committed Appends. Missing or malformed proof falls back to the general ordered diff. Raw Lance writers are outside the supported graph-writer topology, and the certificate is an internal, non-cryptographic capability. The final five-pair production gate passed at 10K (3.875× median; 24,297,472-byte max paired RSS overhead) and 100K (~3.886×; 32,604,160 bytes). | -| MemWAL streaming foundation | ✅ MemWAL system index, durable WAL generations, epoch-fenced shard writer | Internal schema v9 introduced exact identity-keyed physical binding, current-HEAD witness, lifecycle state-v2, per-shard epoch floor, stream-config v3, and the manifest-selected `_stream_tokens.lance` pointer; v10 added the graph-global `stream_profile` singleton and v11 added checked protocol-v2 profile authority. V12 activates lifecycle-v3 fixed-size binding/management/claim chain references plus authenticated WAL-tail authority. Its hidden recovery-v14 path enrolls one empty unsharded shard, recovery-covers every cold writer claim, authenticates each bounded WAL suffix and full active-generation LWW projection, folds exact base/token participants, and quiesces empty or non-empty lanes through restartable `OPEN → DRAINING → SEALED`. V13's recovery-v15 path adds revision-fenced `SEALED → OPEN` resume and guarded `DRAINING → OPEN` abort, claiming a higher epoch and publishing exact terminal claim/management receipts. V14's recovery-v16 path adds the checked-runtime, main-only, same-binding `SEALED` EnsureIndices bridge; v15's recovery-v17 path adds the distinct checked `SEALED` Optimize bridge; v16's recovery-v18 path adds crate-private physical rebind from one exact `SEALED` binding scope to a fresh empty scope that remains `SEALED`; v17's recovery-v19 path adds irreversible stopped/offline authority retirement for an exact `DISABLED`, all-`SEALED`, settled-recovery graph with at least one current `WITHDRAWN` token; v18's recovery-v20 path adds stopped/offline, exact-cut `DataBlock` correction with durable correction/management receipts and explicit `WITHDRAWN` sequencing authority; v19's recovery-v21 path adds deterministic bounded mixed/all-diverted terminal folds, current token-schema-v3 `DEAD_LETTERED`, and retirement over `WITHDRAWN | DEAD_LETTERED`. A restart after terminal claim reclaims with a higher epoch; a restart after physical seal reuses the receipt-bound flushed cut instead of invalidating it. The private B1 worker remains one root-singleflight, no-roll generation capped at 8,192 rows / 32 MiB of logical dense-slice Arrow data with watcher-plus-post-fence acknowledgement; backing-buffer capacity and physical RSS are not admission authority. Private B2 admission adds canonical payload/token digests, trusted hidden metadata, same-key compare-and-chain/idempotency classification, same-generation overlays, and authority recapture after shared admission. The private B2a profile remains unbounded retain-all: no OmniGraph byte/object/file/history quota, no canonical durable `_mem_wal` deletion, typed provider failure, and inert retained orphan residue. The 1/8/32/128 local/RustFS instrument remains advisory. It performs no generation GC or fresh reads. Profile change is cluster-apply-only under explicit offline confirmation and the persisted state lock; `stream_manage` is checked against both the currently applied and desired graph policies. Enabled-profile Mutation/Load/delete require the exact checked cluster serving runtime, while BranchMerge remains closed under both `ENABLED` and `DISABLING` even with that runtime. The exact-`ENABLED` resident owner now also continues unblocked goal-`SEALED` drains, while checked offline disable persists `DISABLING`, drains its finite manifest-derived node-before-edge lane cut serially, adopts `OPEN_AFTER_FOLD` through recovery-v14, and parks on `DataBlock` until correction plus apply retry. `RETIRED` decodes fail-closed and leaves only query/status/export on the source. F6b1's hidden checked path re-proves terminal authority and freezes exact export versions. Ordinary enrolled `DISABLED` ambient export refuses; the receipt-verified ambient `RETIRED` direct-export bridge remains, while F6b5 routes served export through the checked cut. Both paths serialize against destructive root control with the same nonwaiting slot. The cluster-only `stream retire-for-rebuild plan|confirm` handshake is active. Checked offline disable is the sole supported production quiescence owner. F6a adds a failpoints-only advisory driver snapshot and hidden in-process candidate-runtime composition without changing public durable status. F7a activates graph-native served row ingestion with remote-client/CLI/OpenAPI parity while keeping physical lane enrollment private. Its strong graph-authority precondition is derived from existing manifest/runtime evidence, and it reuses the current lazy prepare and resident driver without a new format or coordinator. F7b exposes F6b6's checked cut through graph-redacted HTTP/OpenAPI and remote CLI status. General resume/abort, rebind, lifecycle/maintenance transport, and direct-SDK status contracts remain inactive; exact-terminal served export remains the F6b5 exception, while cluster-only `stream block show|correct` and `stream dead-letter list|export` are the active correction/inspection surfaces. F6b3 adds exact-selected uncovered-tail current-token hit/miss and terminal-page cost evidence without a format or production surface. F6b7 adds the paired failpoints-only selected-index refresh and receipt-key/current-token decision evidence without format, recovery, or production behavior. F6b5 performs pre-header authority/filter/limit validation, incrementally scans exact pinned Lance versions with approximate targets, emits independently owned 64-KiB chunks through a two-chunk queue, reserves the complete per-response queue envelope from one true process-wide budget, and retains both the queue lease and move-only cut through disconnect or a terminal frame. F6b7's uncompacted-profile-cycle bounded NO-GO schedules no standalone production token-index reconciler; direct-SDK status and remaining guardrail acceptance stay open. | +| MemWAL streaming foundation | ✅ MemWAL system index, durable WAL generations, epoch-fenced shard writer | Internal schema v9 introduced exact identity-keyed physical binding, current-HEAD witness, lifecycle state-v2, per-shard epoch floor, stream-config v3, and the manifest-selected `_stream_tokens.lance` pointer; v10 added the graph-global `stream_profile` singleton and v11 added checked protocol-v2 profile authority. V12 activates lifecycle-v3 fixed-size binding/management/claim chain references plus authenticated WAL-tail authority. Its hidden recovery-v14 path enrolls one empty unsharded shard, recovery-covers every cold writer claim, authenticates each bounded WAL suffix and full active-generation LWW projection, folds exact base/token participants, and quiesces empty or non-empty lanes through restartable `OPEN → DRAINING → SEALED`. V13's recovery-v15 path adds revision-fenced `SEALED → OPEN` resume and guarded `DRAINING → OPEN` abort, claiming a higher epoch and publishing exact terminal claim/management receipts. V14's recovery-v16 path adds the checked-runtime, main-only, same-binding `SEALED` EnsureIndices bridge; v15's recovery-v17 path adds the distinct checked `SEALED` Optimize bridge; v16's recovery-v18 path adds crate-private physical rebind from one exact `SEALED` binding scope to a fresh empty scope that remains `SEALED`; v17's recovery-v19 path adds irreversible stopped/offline authority retirement for an exact `DISABLED`, all-`SEALED`, settled-recovery graph with at least one current `WITHDRAWN` token; v18's recovery-v20 path adds stopped/offline, exact-cut `DataBlock` correction with durable correction/management receipts and explicit `WITHDRAWN` sequencing authority; v19's recovery-v21 path adds deterministic bounded mixed/all-diverted terminal folds, current token-schema-v3 `DEAD_LETTERED`, and retirement over `WITHDRAWN | DEAD_LETTERED`. A restart after terminal claim reclaims with a higher epoch; a restart after physical seal reuses the receipt-bound flushed cut instead of invalidating it. The private B1 worker remains one root-singleflight, no-roll generation capped at 8,192 rows / 32 MiB of logical dense-slice Arrow data with watcher-plus-post-fence acknowledgement; backing-buffer capacity and physical RSS are not admission authority. Private B2 admission adds canonical payload/token digests, trusted hidden metadata, same-key compare-and-chain/idempotency classification, same-generation overlays, and authority recapture after shared admission. The private B2a profile remains unbounded retain-all: no OmniGraph byte/object/file/history quota, no canonical durable `_mem_wal` deletion, typed provider failure, and inert retained orphan residue. The 1/8/32/128 local/RustFS instrument remains advisory. It performs no generation GC or fresh reads. Profile change is cluster-apply-only under explicit offline confirmation and the persisted state lock; `stream_manage` is checked against both the currently applied and desired graph policies. Enabled-profile Mutation/Load/delete require the exact checked cluster serving runtime, while BranchMerge remains closed under both `ENABLED` and `DISABLING` even with that runtime. The exact-`ENABLED` resident owner now also continues unblocked goal-`SEALED` drains, while checked offline disable persists `DISABLING`, drains its finite manifest-derived node-before-edge lane cut serially, adopts `OPEN_AFTER_FOLD` through recovery-v14, and parks on `DataBlock` until correction plus apply retry. `RETIRED` decodes fail-closed and leaves only query/status/export on the source. F6b1's hidden checked path re-proves terminal authority and freezes exact export versions. Ordinary enrolled `DISABLED` ambient export refuses; the receipt-verified ambient `RETIRED` direct-export bridge remains, while F6b5 routes served export through the checked cut. Both paths serialize against destructive root control with the same nonwaiting slot. The cluster-only `stream retire-for-rebuild plan|confirm` handshake is active. Checked offline disable is the sole supported production quiescence owner. F6a adds a failpoints-only advisory driver snapshot and hidden in-process candidate-runtime composition without changing public durable status. F7a activates graph-native served row ingestion with remote-client/CLI/OpenAPI parity while keeping physical lane enrollment private. Its strong graph-authority precondition is derived from existing manifest/runtime evidence, and it reuses the current lazy prepare and resident driver without a new format or coordinator. F7b exposes F6b6's checked cut through graph-redacted HTTP/OpenAPI and remote CLI status. F7c exposes selector-free graph-wide resume and checked SEALED EnsureIndices/Optimize by composing recovery-v15/v16/v17; per-declaration resume/abort, rebind, general lifecycle transport, and direct-SDK status contracts remain inactive; exact-terminal served export remains the F6b5 exception, while cluster-only `stream block show|correct` and `stream dead-letter list|export` are the active correction/inspection surfaces. F6b3 adds exact-selected uncovered-tail current-token hit/miss and terminal-page cost evidence without a format or production surface. F6b7 adds the paired failpoints-only selected-index refresh and receipt-key/current-token decision evidence without format, recovery, or production behavior. F6b5 performs pre-header authority/filter/limit validation, incrementally scans exact pinned Lance versions with approximate targets, emits independently owned 64-KiB chunks through a two-chunk queue, reserves the complete per-response queue envelope from one true process-wide budget, and retains both the queue lease and move-only cut through disconnect or a terminal frame. F6b7's uncompacted-profile-cycle bounded NO-GO schedules no standalone production token-index reconciler; direct-SDK status and remaining guardrail acceptance stay open. | | Vector search | ✅ | `nearest()` query op; embedding pipeline (Gemini / OpenAI clients); `@embed` in schema | | Full-text search | ✅ | `search/fuzzy/match_text/bm25` query ops | | Hybrid ranking | — | `rrf(...)` Reciprocal Rank Fusion in one runtime | diff --git a/crates/omnigraph-api-types/src/lib.rs b/crates/omnigraph-api-types/src/lib.rs index 79c0e505..97983b05 100644 --- a/crates/omnigraph-api-types/src/lib.rs +++ b/crates/omnigraph-api-types/src/lib.rs @@ -5,8 +5,9 @@ use omnigraph::db::{ GraphCommit, GraphStreamDeclaration, GraphStreamDeclarationStatus, - GraphStreamDriverErrorStatus, GraphStreamDriverStatus, GraphStreamOperationalStatus, - GraphStreamPendingStatus, GraphStreamRebuildBlocker, GraphStreamRebuildStatus, + GraphStreamDriverErrorStatus, GraphStreamDriverStatus, GraphStreamEnsureIndicesResult, + GraphStreamOperationalStatus, GraphStreamOptimizeResult, GraphStreamPendingStatus, + GraphStreamRebuildBlocker, GraphStreamRebuildStatus, GraphStreamResumeResult, GraphStreamTokenCounts, MergeOutcome, ReadTarget, SchemaApplyResult, Snapshot, }; use omnigraph::error::{MergeConflict, MergeConflictKind}; @@ -825,6 +826,61 @@ pub struct StreamStatusOutput { pub rebuild: StreamRebuildStatusOutput, } +/// Aggregate result of reopening every sealed streaming declaration in a +/// graph. Declaration, table, lane, dataset, and recovery identities are +/// deliberately absent from this graph-level control-plane shape. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct StreamResumeOutput { + pub profile_revision: u64, + pub enrolled_declarations: u64, + pub resumed_declarations: u64, + pub already_open_declarations: u64, +} + +/// Aggregate result of graph-wide checked index refresh. Any enrolled +/// declaration changed by the operation is required to be sealed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct StreamEnsureIndicesOutput { + pub changed: bool, + pub pending_index_count: u64, +} + +/// Aggregate result of graph-wide checked stream optimization. Any enrolled +/// declaration changed by the operation is required to be sealed. Physical +/// fragment and dataset details stay inside the engine. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct StreamOptimizeOutput { + pub changed: bool, + pub pending_index_count: u64, + pub requires_repair: bool, +} + +pub fn stream_resume_output(value: GraphStreamResumeResult) -> StreamResumeOutput { + StreamResumeOutput { + profile_revision: value.profile_revision, + enrolled_declarations: value.enrolled_declarations, + resumed_declarations: value.resumed_declarations, + already_open_declarations: value.already_open_declarations, + } +} + +pub fn stream_ensure_indices_output( + value: GraphStreamEnsureIndicesResult, +) -> StreamEnsureIndicesOutput { + StreamEnsureIndicesOutput { + changed: value.changed, + pending_index_count: value.pending_index_count, + } +} + +pub fn stream_optimize_output(value: GraphStreamOptimizeResult) -> StreamOptimizeOutput { + StreamOptimizeOutput { + changed: value.changed, + pending_index_count: value.pending_index_count, + requires_repair: value.requires_repair, + } +} + fn stream_profile_mode_output( value: &str, ) -> std::result::Result { diff --git a/crates/omnigraph-cli/src/cli.rs b/crates/omnigraph-cli/src/cli.rs index 02c17b0f..4b10a13f 100644 --- a/crates/omnigraph-cli/src/cli.rs +++ b/crates/omnigraph-cli/src/cli.rs @@ -16,7 +16,7 @@ pub(crate) const DEFAULT_BEARER_TOKEN_ENV: &str = "OMNIGRAPH_BEARER_TOKEN"; COMMANDS BY CAPABILITY:\n \ any — run against a graph, served (--server / --profile) or embedded (--store / a \ URI): query, mutate, load, branch, snapshot, export, commit, schema show/apply.\n \ -served — require a server: stream ingest/status (graph scope) and graphs (registry scope).\n \ +served — require a server: stream ingest/status/resume/maintenance (graph scope) and graphs (registry scope).\n \ direct — direct storage access; reject --server (init, optimize, repair, cleanup, \ schema plan, lint).\n \ control — manage or inspect a cluster (cluster via --config; policy & queries via \ @@ -394,6 +394,34 @@ pub(crate) enum StreamCommand { #[arg(long)] json: bool, }, + /// Reopen every sealed declaration in the served graph. + Resume { + /// Emit the graph-level result as JSON. + #[arg(long)] + json: bool, + }, + /// Run checked graph-wide maintenance; any enrolled declaration changed + /// by the operation must be sealed. + Maintenance { + #[command(subcommand)] + command: StreamMaintenanceCommand, + }, +} + +#[derive(Debug, Subcommand)] +pub(crate) enum StreamMaintenanceCommand { + /// Reconcile declared indexes across the graph. + EnsureIndices { + /// Emit the graph-level result as JSON. + #[arg(long)] + json: bool, + }, + /// Compact the graph through one coordinated publish. + Optimize { + /// Emit the graph-level result as JSON. + #[arg(long)] + json: bool, + }, } #[derive(Debug, Subcommand)] @@ -566,8 +594,6 @@ pub(crate) enum StreamDeadLetterCommand { pub(crate) enum StreamBlockCommand { /// Revalidate and print one bounded page of correction evidence. Show { - /// Exact manifest table key, for example node:Person. - table_key: String, /// Cluster config directory containing cluster.yaml. #[arg(long, default_value = ".")] config: PathBuf, @@ -586,8 +612,6 @@ pub(crate) enum StreamBlockCommand { }, /// Apply one ordered REPLACE/WITHDRAW plan to the exact blocked cut. Correct { - /// Exact manifest table key, for example node:Person. - table_key: String, /// Cluster config directory containing cluster.yaml. #[arg(long, default_value = ".")] config: PathBuf, diff --git a/crates/omnigraph-cli/src/client.rs b/crates/omnigraph-cli/src/client.rs index f43477eb..7901e5a0 100644 --- a/crates/omnigraph-cli/src/client.rs +++ b/crates/omnigraph-cli/src/client.rs @@ -34,8 +34,8 @@ use omnigraph_api_types::{ ErrorOutput, ExportRequest, GraphListResponse, IngestOutput, IngestRequest, InvokeStoredQueryRequest, ReadOutput, ReadRequest, SchemaApplyOutput, SchemaApplyRequest, SchemaOutput, SnapshotOutput, - StreamStatusOutput, commit_output, ingest_output, read_output, schema_apply_output, - snapshot_payload, + StreamEnsureIndicesOutput, StreamOptimizeOutput, StreamResumeOutput, StreamStatusOutput, + commit_output, ingest_output, read_output, schema_apply_output, snapshot_payload, }; use omnigraph_compiler::catalog::Catalog; use reqwest::Method; @@ -341,6 +341,18 @@ impl GraphClient { .await } + /// Resolve one graph-wide served stream control. The public surface has no + /// declaration/table selector and never accepts a client-supplied actor. + pub(crate) async fn resolve_stream_control( + command: &str, + server: Option<&str>, + graph: Option<&str>, + profile: Option<&str>, + store: Option<&str>, + ) -> Result { + Self::resolve_selected_served_graph(command, server, graph, None, profile, store).await + } + /// Shared graph-selection owner for the served-only stream family. /// `command` is threaded only into the observable missing-graph error; /// keeping the ingest spelling here preserves its existing contract. @@ -1071,6 +1083,79 @@ impl GraphClient { } } + /// Reopen every currently sealed declaration in the selected graph. + pub(crate) async fn stream_resume(&self) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::POST, + remote_url(base_url, &["stream", "resume"], &[])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { .. } => bail!( + "internal error: `stream resume` reached an embedded client — stream controls \ + always resolve a server" + ), + } + } + + /// Reconcile declared indexes across the graph. Any enrolled declaration + /// changed by the operation must be sealed; physical datasets stay private + /// behind the graph coordinator. + pub(crate) async fn stream_ensure_indices(&self) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::POST, + remote_url(base_url, &["stream", "maintenance", "ensure-indices"], &[])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { .. } => bail!( + "internal error: `stream maintenance ensure-indices` reached an embedded client" + ), + } + } + + /// Compact the graph through the existing coordinated publish. Any + /// enrolled declaration changed by the operation must be sealed. + pub(crate) async fn stream_optimize(&self) -> Result { + match self { + GraphClient::Remote { + http, + base_url, + token, + } => { + remote_json( + http, + Method::POST, + remote_url(base_url, &["stream", "maintenance", "optimize"], &[])?, + None, + token.as_deref(), + ) + .await + } + GraphClient::Embedded { .. } => bail!( + "internal error: `stream maintenance optimize` reached an embedded client" + ), + } + } + /// `export` — stream the branch as JSONL into `writer`. The streaming /// shape (a `W: Write`, not a returned DTO) is why this lands in 3c /// rather than 3b. Opens WITHOUT policy (like reads), so it is reached @@ -1516,6 +1601,69 @@ mod tests { server.await.unwrap(); } + #[tokio::test] + async fn graph_stream_controls_post_bodyless_aggregate_requests() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}/graphs/knowledge", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + for (path, response) in [ + ( + "/graphs/knowledge/stream/resume", + r#"{"profile_revision":4,"enrolled_declarations":3,"resumed_declarations":2,"already_open_declarations":1}"#, + ), + ( + "/graphs/knowledge/stream/maintenance/ensure-indices", + r#"{"changed":true,"pending_index_count":0}"#, + ), + ( + "/graphs/knowledge/stream/maintenance/optimize", + r#"{"changed":false,"pending_index_count":1,"requires_repair":false}"#, + ), + ] { + let (mut stream, _) = listener.accept().await.unwrap(); + let request = read_http_request(&mut stream).await; + assert!( + request + .head + .starts_with(&format!("POST {path} HTTP/1.1")), + "{}", + request.head + ); + assert!( + request + .head + .to_ascii_lowercase() + .contains("authorization: bearer manage-token") + ); + assert!(request.body.is_empty()); + write_response( + &mut stream, + "200 OK", + &[("Content-Type", "application/json")], + response.as_bytes(), + ) + .await; + } + }); + + let client = GraphClient::Remote { + http: reqwest::Client::new(), + base_url, + token: Some("manage-token".to_string()), + }; + let resumed = client.stream_resume().await.unwrap(); + assert_eq!(resumed.resumed_declarations, 2); + assert_eq!(resumed.already_open_declarations, 1); + let indexed = client.stream_ensure_indices().await.unwrap(); + assert!(indexed.changed); + assert_eq!(indexed.pending_index_count, 0); + let optimized = client.stream_optimize().await.unwrap(); + assert!(!optimized.changed); + assert_eq!(optimized.pending_index_count, 1); + assert!(!optimized.requires_repair); + server.await.unwrap(); + } + #[tokio::test] async fn stream_commands_require_and_resolve_a_selected_graph_without_network_io() { let error = match GraphClient::resolve_stream_ingest( @@ -1564,5 +1712,22 @@ mod tests { .await .unwrap(); assert_eq!(client.uri(), "http://127.0.0.1:9/graphs/knowledge"); + + let error = match GraphClient::resolve_stream_control( + "stream resume", + Some("http://127.0.0.1:9"), + None, + None, + None, + ) + .await + { + Ok(_) => panic!("stream resume accepted a server without a selected graph"), + Err(error) => error.to_string(), + }; + assert!( + error.contains("`stream resume` requires one selected graph"), + "{error}" + ); } } diff --git a/crates/omnigraph-cli/src/main.rs b/crates/omnigraph-cli/src/main.rs index 4160767a..e005c337 100644 --- a/crates/omnigraph-cli/src/main.rs +++ b/crates/omnigraph-cli/src/main.rs @@ -9,9 +9,9 @@ use omnigraph::error::OmniError; use omnigraph::loader::LoadMode; use omnigraph_api_types::{ ChangeOutput, CommitOutput, ErrorOutput, IngestOutput, ReadOutput, SchemaApplyOutput, - SnapshotTableOutput, StreamDriverStateOutput, StreamIngestKindOutput, StreamLifecycleOutput, - StreamPendingStatusOutput, StreamProfileModeOutput, StreamRebuildBlockerOutput, - StreamStatusOutput, + SnapshotTableOutput, StreamDriverStateOutput, StreamEnsureIndicesOutput, + StreamIngestKindOutput, StreamLifecycleOutput, StreamOptimizeOutput, StreamPendingStatusOutput, + StreamProfileModeOutput, StreamRebuildBlockerOutput, StreamResumeOutput, StreamStatusOutput, }; use omnigraph_cluster::{ ApplyOptions, ApplyOutput, ApproveOutput, DiagnosticSeverity, ForceUnlockOutput, PlanOutput, @@ -759,6 +759,57 @@ async fn main() -> Result<()> { let output = client.stream_operational_status().await?; finish_stream_status(&output, json)?; } + StreamCommand::Resume { json } => { + let client = client::GraphClient::resolve_stream_control( + "stream resume", + cli.server.as_deref(), + cli.graph.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + echo_write_target(cli.quiet, "stream resume", client.uri(), true); + let output = client.stream_resume().await?; + finish_stream_resume(&output, json)?; + } + StreamCommand::Maintenance { command } => match command { + StreamMaintenanceCommand::EnsureIndices { json } => { + let client = client::GraphClient::resolve_stream_control( + "stream maintenance ensure-indices", + cli.server.as_deref(), + cli.graph.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + echo_write_target( + cli.quiet, + "stream maintenance ensure-indices", + client.uri(), + true, + ); + let output = client.stream_ensure_indices().await?; + finish_stream_ensure_indices(&output, json)?; + } + StreamMaintenanceCommand::Optimize { json } => { + let client = client::GraphClient::resolve_stream_control( + "stream maintenance optimize", + cli.server.as_deref(), + cli.graph.as_deref(), + cli.profile.as_deref(), + cli.store.as_deref(), + ) + .await?; + echo_write_target( + cli.quiet, + "stream maintenance optimize", + client.uri(), + true, + ); + let output = client.stream_optimize().await?; + finish_stream_optimize(&output, json)?; + } + }, }, Command::Query { name, @@ -1232,7 +1283,6 @@ async fn main() -> Result<()> { let actor = resolve_cluster_actor(cli.as_actor.as_deref())?; match command { StreamBlockCommand::Show { - table_key, config, block_token, cursor, @@ -1242,7 +1292,6 @@ async fn main() -> Result<()> { let output = show_stream_data_block_config_dir( config, graph_id, - table_key, block_token, cursor.as_deref(), StreamBlockControlOptions { @@ -1254,7 +1303,6 @@ async fn main() -> Result<()> { finish_stream_block_show(&output, json)?; } StreamBlockCommand::Correct { - table_key, config, block_token, correction_id, @@ -1291,7 +1339,6 @@ async fn main() -> Result<()> { let output = correct_stream_data_block_config_dir( config, graph_id, - table_key, request, StreamBlockControlOptions { actor, diff --git a/crates/omnigraph-cli/src/output.rs b/crates/omnigraph-cli/src/output.rs index 8d17130f..d01897be 100644 --- a/crates/omnigraph-cli/src/output.rs +++ b/crates/omnigraph-cli/src/output.rs @@ -312,6 +312,14 @@ pub(crate) fn render_stream_status_human(output: &StreamStatusOutput) -> String writeln!(rendered, " drain: {} / {}", drain.goal, drain.phase) .expect("writing stream status to a String cannot fail"); } + if let Some(fold) = status.last_fold.as_ref() { + writeln!( + rendered, + " last fold: {} ({} input row(s), {} visible row(s), recorded {})", + fold.outcome, fold.input_rows, fold.visible_rows, fold.recorded_at + ) + .expect("writing stream status to a String cannot fail"); + } } } writeln!( @@ -354,6 +362,10 @@ pub(crate) fn render_stream_status_human(output: &StreamStatusOutput) -> String }; result.expect("writing stream status to a String cannot fail"); } + if let Some(kind) = output.driver.last_completion_kind.as_deref() { + writeln!(rendered, " last completion: {kind}") + .expect("writing stream status to a String cannot fail"); + } if output.rebuild.ready { writeln!(rendered, " rebuild: ready") .expect("writing stream status to a String cannot fail"); @@ -364,6 +376,44 @@ pub(crate) fn render_stream_status_human(output: &StreamStatusOutput) -> String render_stream_rebuild_blocker(&mut rendered, blocker); } } + let next = if output.recovery_pending_count > 0 { + "reopen/restart the graph to resolve recovery before another stream operation" + } else if output + .enrolled_declarations + .iter() + .any(|status| status.strict_block.is_some()) + { + "stop serving, inspect the graph-level block token, correct it, then retry the control" + } else if output + .enrolled_declarations + .iter() + .any(|status| status.lifecycle == StreamLifecycleOutput::Draining) + { + "wait for the drain, or retry the stopped-cluster apply after correcting a block" + } else { + match output.profile_mode { + StreamProfileModeOutput::Enabled + if output + .enrolled_declarations + .iter() + .any(|status| status.lifecycle == StreamLifecycleOutput::Sealed) => + { + "optionally run graph-wide `stream maintenance ensure-indices|optimize`, then `stream resume`" + } + StreamProfileModeOutput::Enabled => { + "send graph rows with `omnigraph stream ingest`" + } + StreamProfileModeOutput::Disabling => { + "stop serving and retry `omnigraph cluster apply` to finish disabling" + } + StreamProfileModeOutput::Disabled => { + "set `streaming: true`, apply the cluster config, and restart the server" + } + StreamProfileModeOutput::Retired => "export and rebuild into a fresh graph", + } + }; + writeln!(rendered, " next: {next}") + .expect("writing stream status to a String cannot fail"); rendered } @@ -380,6 +430,57 @@ pub(crate) fn finish_stream_status(output: &StreamStatusOutput, json: bool) -> R } } +pub(crate) fn finish_stream_resume(output: &StreamResumeOutput, json: bool) -> Result<()> { + if json { + return print_json(output); + } + println!( + "stream resume: {} resumed, {} already open ({} enrolled; profile revision {})", + output.resumed_declarations, + output.already_open_declarations, + output.enrolled_declarations, + output.profile_revision + ); + Ok(()) +} + +pub(crate) fn finish_stream_ensure_indices( + output: &StreamEnsureIndicesOutput, + json: bool, +) -> Result<()> { + if json { + return print_json(output); + } + println!( + "stream index maintenance: {} ({} pending index(es))", + if output.changed { "published" } else { "no change" }, + output.pending_index_count + ); + Ok(()) +} + +pub(crate) fn finish_stream_optimize(output: &StreamOptimizeOutput, json: bool) -> Result<()> { + if json { + return print_json(output); + } + println!( + "stream optimize: {} ({} pending index(es))", + if output.changed { "published" } else { "no change" }, + output.pending_index_count + ); + if output.requires_repair { + println!("{}", stream_optimize_drift_advice()); + } + Ok(()) +} + +fn stream_optimize_drift_advice() -> &'static str { + concat!( + " uncovered storage drift needs offline review; stop serving and preserve the root\n", + " enrolled drift has no supported in-place repair; rebuild a fresh graph from the last verified clean export or backup", + ) +} + pub(crate) fn print_cluster_validate_human(output: &ValidateOutput) { if output.ok { println!( @@ -660,10 +761,9 @@ pub(crate) fn finish_stream_block_show(output: &StreamBlockShowOutput, json: boo print_json(output)?; } else if let Some(page) = output.page.as_ref() { println!("stream data block for {}", output.graph_id); - println!(" table: {}", page.table_key); println!( - " table identity: {}:{}", - page.stable_table_id, page.table_incarnation_id + " declaration: {} {}", + page.declaration.kind, page.declaration.type_name ); println!(" block token: {}", page.block_token); println!(" lifecycle revision: {}", page.lifecycle_revision); @@ -703,7 +803,6 @@ pub(crate) fn finish_stream_dead_letter_list( " source profile revision: {}", page.source_profile_revision ); - println!(" token table version: {}", page.token_table_version); println!(" entries:"); for entry in &page.entries { println!(" {}", serde_json::to_string(entry)?); @@ -739,7 +838,6 @@ pub(crate) fn finish_stream_dead_letter_export( " source profile revision: {}", page.source_profile_revision ); - println!(" token table version: {}", page.token_table_version); println!(" entries:"); for entry in &page.entries { println!(" {}", serde_json::to_string(entry)?); @@ -1453,7 +1551,7 @@ mod tests { use omnigraph_compiler::schema::parser::parse_schema; use std::collections::BTreeMap; - use super::{render_annotations, render_stream_status_human}; + use super::{render_annotations, render_stream_status_human, stream_optimize_drift_advice}; #[test] fn render_annotations_quotes_values_so_embed_round_trips() { @@ -1528,7 +1626,9 @@ mod tests { assert!( rendered.contains("driver: failed (advisory; 5 pending, 6 open fold(s) published)") ); + assert!(rendered.contains("last completion: folded")); assert!(rendered.contains("terminal sequencing authority: 1 withdrawn, 2 dead-lettered")); + assert!(rendered.contains("next: reopen/restart the graph to resolve recovery")); for forbidden in [ "dataset", "table_key", @@ -1544,4 +1644,13 @@ mod tests { ); } } + + #[test] + fn stream_optimize_drift_advice_does_not_promise_enrolled_repair() { + let advice = stream_optimize_drift_advice(); + assert!(advice.contains("stop serving and preserve the root")); + assert!(advice.contains("enrolled drift has no supported in-place repair")); + assert!(advice.contains("rebuild a fresh graph")); + assert!(!advice.contains("omnigraph repair")); + } } diff --git a/crates/omnigraph-cli/src/planes.rs b/crates/omnigraph-cli/src/planes.rs index e570d85b..d6ffa5a1 100644 --- a/crates/omnigraph-cli/src/planes.rs +++ b/crates/omnigraph-cli/src/planes.rs @@ -15,6 +15,7 @@ use color_eyre::eyre::bail; use crate::cli::{ Cli, ClusterCommand, ClusterStreamCommand, Command, GraphsCommand, QueriesCommand, SchemaCommand, StreamBlockCommand, StreamCommand, StreamDeadLetterCommand, + StreamMaintenanceCommand, StreamRetireForRebuildCommand, }; @@ -346,6 +347,13 @@ pub(crate) fn command_label(cmd: &Command) -> &'static str { Command::Stream { command } => match command { StreamCommand::Ingest { .. } => "stream ingest", StreamCommand::Status { .. } => "stream status", + StreamCommand::Resume { .. } => "stream resume", + StreamCommand::Maintenance { command } => match command { + StreamMaintenanceCommand::EnsureIndices { .. } => { + "stream maintenance ensure-indices" + } + StreamMaintenanceCommand::Optimize { .. } => "stream maintenance optimize", + }, }, } } @@ -547,7 +555,6 @@ mod tests { "stream", "block", "show", - "node:Person", "--block-token", "block-1", ]), @@ -579,6 +586,15 @@ mod tests { assert_eq!(cap(&["omnigraph", "graphs", "list"]), Capability::Served); assert_eq!(cap(&["omnigraph", "stream", "ingest"]), Capability::Served); assert_eq!(cap(&["omnigraph", "stream", "status"]), Capability::Served); + assert_eq!(cap(&["omnigraph", "stream", "resume"]), Capability::Served); + assert_eq!( + cap(&["omnigraph", "stream", "maintenance", "ensure-indices"]), + Capability::Served + ); + assert_eq!( + cap(&["omnigraph", "stream", "maintenance", "optimize"]), + Capability::Served + ); assert_eq!(cap(&["omnigraph", "alias", "who"]), Capability::Local); assert_eq!( cap(&["omnigraph", "optimize", "graph.omni"]), @@ -606,7 +622,6 @@ mod tests { "stream", "block", "correct", - "node:Person", "--block-token", "block-1", "--correction-id", @@ -719,6 +734,42 @@ mod tests { } => assert!(json), other => panic!("expected stream status, got {other:?}"), } + + for args in [ + vec!["omnigraph", "stream", "resume"], + vec!["omnigraph", "stream", "maintenance", "ensure-indices"], + vec!["omnigraph", "stream", "maintenance", "optimize"], + ] { + let mut served = args.clone(); + served.extend([ + "--server", + "http://server.invalid:9", + "--graph", + "knowledge", + "--json", + ]); + let cli = Cli::try_parse_from(served).unwrap(); + guard_addressing(&cli).unwrap(); + + let mut direct = args; + direct.extend(["--store", "file:///must-not-open.omni"]); + let cli = Cli::try_parse_from(direct).unwrap(); + let error = guard_addressing(&cli).unwrap_err().to_string(); + assert!(error.contains("is a served command"), "{error}"); + assert!(error.contains("--store addresses"), "{error}"); + } + + assert!( + Cli::try_parse_from([ + "omnigraph", + "stream", + "resume", + "--type", + "Person", + ]) + .is_err(), + "graph resume must not accept a declaration selector" + ); } #[test] diff --git a/crates/omnigraph-cli/tests/cli_cluster.rs b/crates/omnigraph-cli/tests/cli_cluster.rs index 0bdad871..8d85a2eb 100644 --- a/crates/omnigraph-cli/tests/cli_cluster.rs +++ b/crates/omnigraph-cli/tests/cli_cluster.rs @@ -1228,7 +1228,6 @@ fn cluster_stream_block_requires_graph_and_accepts_only_config_scope() { .arg("stream") .arg("block") .arg("show") - .arg("node:Person") .arg("--config") .arg(temp.path()) .arg("--block-token") @@ -1250,7 +1249,6 @@ fn cluster_stream_block_requires_graph_and_accepts_only_config_scope() { .arg("stream") .arg("block") .arg("show") - .arg("node:Person") .arg("--config") .arg(temp.path()) .arg("--block-token") @@ -1275,7 +1273,6 @@ fn cluster_stream_block_reports_structured_offline_preflight_errors() { .arg("stream") .arg("block") .arg("show") - .arg("node:Person") .arg("--config") .arg(temp.path()) .arg("--block-token") @@ -1311,7 +1308,6 @@ fn cluster_stream_block_correct_rejects_unknown_plan_fields_before_preflight() { .arg("stream") .arg("block") .arg("correct") - .arg("node:Person") .arg("--config") .arg(temp.path()) .arg("--block-token") @@ -1331,3 +1327,24 @@ fn cluster_stream_block_correct_rejects_unknown_plan_fields_before_preflight() { "plan parsing must precede cluster preflight" ); } + +#[test] +fn cluster_stream_block_rejects_the_old_per_table_selector() { + let output = output_failure( + cli() + .arg("--graph") + .arg("knowledge") + .arg("cluster") + .arg("stream") + .arg("block") + .arg("show") + .arg("node:Person") + .arg("--block-token") + .arg("block-1"), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("unexpected argument 'node:Person'"), + "{stderr}" + ); +} diff --git a/crates/omnigraph-cli/tests/system_local.rs b/crates/omnigraph-cli/tests/system_local.rs index f81398f7..5482c09d 100644 --- a/crates/omnigraph-cli/tests/system_local.rs +++ b/crates/omnigraph-cli/tests/system_local.rs @@ -1870,6 +1870,262 @@ fn skip_system_e2e(test_name: &str) -> bool { false } +fn write_firehose_cluster_config(dir: &std::path::Path, streaming: bool) { + std::fs::write( + dir.join("cluster.yaml"), + format!( + r#" +version: 1 +state: + backend: cluster + lock: true +graphs: + knowledge: + schema: ./firehose.pg + streaming: {streaming} + queries: + streamed_edge: + file: ./firehose.gq +"# + ), + ) + .unwrap(); +} + +fn stream_cli_json(base_url: &str, args: &[&str]) -> serde_json::Value { + let mut command = cli(); + command + .arg("--server") + .arg(base_url) + .arg("--graph") + .arg("knowledge") + .arg("stream"); + for arg in args { + command.arg(arg); + } + command.arg("--json"); + parse_stdout_json(&output_success(&mut command)) +} + +fn ingest_firehose_round(dir: &std::path::Path, base_url: &str, round: u64) -> (i64, i64) { + let first_id = format!("firehose-{round}-a"); + let second_id = format!("firehose-{round}-b"); + let edge_id = format!("firehose-{round}-edge"); + let first_score = (round * 10 + 1) as i64; + let second_score = first_score + 1; + let write_id = |ordinal: u64| format!("70000000-0000-4000-8000-{:012x}", round * 3 + ordinal); + let rows = [ + serde_json::json!({ + "type": "Person", + "data": {"id": first_id, "score": first_score}, + "$stream": {"write_id": write_id(0), "predecessor_token": null}, + }), + serde_json::json!({ + "type": "Person", + "data": {"id": second_id, "score": second_score}, + "$stream": {"write_id": write_id(1), "predecessor_token": null}, + }), + serde_json::json!({ + "edge": "Knows", + "from": first_id, + "to": second_id, + "data": {"id": edge_id}, + "$stream": {"write_id": write_id(2), "predecessor_token": null}, + }), + ]; + let mut input = rows + .iter() + .map(|row| serde_json::to_string(row).unwrap()) + .collect::>() + .join("\n"); + input.push('\n'); + let data = dir.join(format!("firehose-round-{round}.ndjson")); + write_jsonl(&data, &input); + + let output = output_success( + cli() + .arg("--server") + .arg(base_url) + .arg("--graph") + .arg("knowledge") + .arg("stream") + .arg("ingest") + .arg("--data") + .arg(&data), + ); + let outcomes = stdout_string(&output) + .lines() + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + assert_eq!(outcomes.len(), 3, "unexpected ingest output: {outcomes:?}"); + for (ordinal, outcome) in outcomes.iter().enumerate() { + assert_eq!(outcome["ordinal"], ordinal as u64, "{outcome}"); + assert_eq!(outcome["status"], "durable", "{outcome}"); + assert_eq!(outcome["scope"], "row", "{outcome}"); + assert!(outcome["stream_token"].is_string(), "{outcome}"); + } + (first_score, second_score) +} + +fn wait_for_streamed_edge(base_url: &str, first_score: i64, second_score: i64) { + let client = Client::new(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + let (status, body) = invoke_query( + &client, + base_url, + "knowledge", + "streamed_edge", + serde_json::json!({"score": first_score}), + ); + assert_eq!(status, 200, "{body}"); + if body["row_count"] == 1 { + assert_eq!(body["rows"][0]["p.score"], first_score, "{body}"); + assert_eq!(body["rows"][0]["f.score"], second_score, "{body}"); + return; + } + assert!( + std::time::Instant::now() < deadline, + "resident fold driver did not publish streamed edge {first_score}->{second_score}: {body}" + ); + std::thread::sleep(std::time::Duration::from_millis(25)); + } +} + +fn assert_stream_lifecycles(status: &serde_json::Value, expected: &str) { + let declarations = status["enrolled_declarations"].as_array().unwrap(); + assert_eq!(declarations.len(), 2, "{status}"); + assert!( + declarations + .iter() + .all(|declaration| declaration["lifecycle"] == expected), + "{status}" + ); + let logical_declarations = declarations + .iter() + .map(|declaration| { + ( + declaration["kind"].as_str().unwrap(), + declaration["type"].as_str().unwrap(), + ) + }) + .collect::>(); + assert_eq!( + logical_declarations, + std::collections::BTreeSet::from([("edge", "Knows"), ("node", "Person")]), + "{status}" + ); +} + +/// The public firehose journey through the real binaries: cluster-owned +/// enablement, graph-native CLI ingest, automatic node/edge folds, an offline +/// disable/re-enable handoff, productive sealed maintenance, graph-wide +/// resume, and a successful successor ingest. The test deliberately never +/// names a physical table, dataset, or lane. +#[test] +fn local_cluster_firehose_golden_journey_uses_graph_only_controls() { + if skip_system_e2e("local_cluster_firehose_golden_journey_uses_graph_only_controls") { + return; + } + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); + std::fs::write( + dir.join("firehose.pg"), + "node Person { score: I32 }\nedge Knows: Person -> Person\n", + ) + .unwrap(); + std::fs::write( + dir.join("firehose.gq"), + r#" +query streamed_edge($score: I32) { + match { + $p: Person { score: $score } + $p knows $f + } + return { $p.score, $f.score } +} +"#, + ) + .unwrap(); + write_firehose_cluster_config(dir, true); + + assert_eq!(cluster_json(dir, "import")["ok"], true); + let enabled = cluster_json(dir, "apply"); + assert_eq!(enabled["ok"], true, "{enabled}"); + assert_eq!(enabled["converged"], true, "{enabled}"); + + // Each round waits for graph visibility before the next request. That + // forces four independently published node/edge fragments, so the later + // sealed EnsureIndices and Optimize calls exercise productive work rather + // than merely proving their no-op routing. + { + let server = spawn_server_with_cluster(dir); + for round in 0..4 { + let (first_score, second_score) = ingest_firehose_round(dir, &server.base_url, round); + wait_for_streamed_edge(&server.base_url, first_score, second_score); + } + + let status = stream_cli_json(&server.base_url, &["status"]); + assert_eq!(status["profile_mode"], "enabled", "{status}"); + assert_eq!(status["token_counts"]["present"], 12, "{status}"); + assert_eq!(status["recovery_pending_count"], 0, "{status}"); + assert_eq!(status["driver"]["state"], "running", "{status}"); + assert_stream_lifecycles(&status, "open"); + } + + // The server process is gone before either profile change. Explicit false + // drains every initialized declaration to SEALED; true re-enables the + // profile without reopening those declarations as a side effect. + write_firehose_cluster_config(dir, false); + let disabled = cluster_json(dir, "apply"); + assert_eq!(disabled["ok"], true, "{disabled}"); + assert_eq!(disabled["converged"], true, "{disabled}"); + + write_firehose_cluster_config(dir, true); + let reenabled = cluster_json(dir, "apply"); + assert_eq!(reenabled["ok"], true, "{reenabled}"); + assert_eq!(reenabled["converged"], true, "{reenabled}"); + + let server = spawn_server_with_cluster(dir); + let sealed = stream_cli_json(&server.base_url, &["status"]); + assert_eq!(sealed["profile_mode"], "enabled", "{sealed}"); + assert_eq!(sealed["recovery_pending_count"], 0, "{sealed}"); + assert_stream_lifecycles(&sealed, "sealed"); + + let indices = stream_cli_json(&server.base_url, &["maintenance", "ensure-indices"]); + assert_eq!(indices["changed"], true, "{indices}"); + assert_eq!(indices["pending_index_count"], 0, "{indices}"); + + let optimized = stream_cli_json(&server.base_url, &["maintenance", "optimize"]); + assert_eq!(optimized["changed"], true, "{optimized}"); + assert_eq!(optimized["pending_index_count"], 0, "{optimized}"); + assert_eq!(optimized["requires_repair"], false, "{optimized}"); + + let after_maintenance = stream_cli_json(&server.base_url, &["status"]); + assert_stream_lifecycles(&after_maintenance, "sealed"); + + let resumed = stream_cli_json(&server.base_url, &["resume"]); + assert_eq!(resumed["enrolled_declarations"], 2, "{resumed}"); + assert_eq!(resumed["resumed_declarations"], 2, "{resumed}"); + assert_eq!(resumed["already_open_declarations"], 0, "{resumed}"); + let retry = stream_cli_json(&server.base_url, &["resume"]); + assert_eq!(retry["resumed_declarations"], 0, "{retry}"); + assert_eq!(retry["already_open_declarations"], 2, "{retry}"); + + let open = stream_cli_json(&server.base_url, &["status"]); + assert_stream_lifecycles(&open, "open"); + let (first_score, second_score) = ingest_firehose_round(dir, &server.base_url, 4); + wait_for_streamed_edge(&server.base_url, first_score, second_score); + let final_status = stream_cli_json(&server.base_url, &["status"]); + assert_eq!( + final_status["token_counts"]["present"], 15, + "{final_status}" + ); + assert_eq!(final_status["recovery_pending_count"], 0, "{final_status}"); + assert_stream_lifecycles(&final_status, "open"); +} + /// The whole control-plane story in one test: declare two graphs → converge /// (apply creates them) → serve → evolve schema+query in one apply → restart /// serves the new shape → out-of-band drift converged back → approved graph diff --git a/crates/omnigraph-cluster/src/stream_block.rs b/crates/omnigraph-cluster/src/stream_block.rs index dde6a4d0..a618c884 100644 --- a/crates/omnigraph-cluster/src/stream_block.rs +++ b/crates/omnigraph-cluster/src/stream_block.rs @@ -61,10 +61,155 @@ impl OfflineStreamControlKind { } } +fn redacted_stream_control_error( + kind: OfflineStreamControlKind, + error: &omnigraph::error::OmniError, +) -> &'static str { + use omnigraph::error::{ManifestErrorKind, OmniError}; + + match error { + OmniError::Policy(_) => "graph stream control was denied by policy", + OmniError::StreamingAuthorityMismatch { .. } + | OmniError::StreamingRequiresClusterRuntime { .. } => { + "offline graph streaming authority is unavailable or changed; stop every live runtime, refresh cluster state, and retry" + } + OmniError::ResourceLimitExceeded { .. } + | OmniError::Manifest(omnigraph::error::ManifestError { + kind: ManifestErrorKind::BadRequest, + .. + }) => "graph stream control request is invalid or exceeds a safety limit", + OmniError::StreamLifecycleChanged { .. } + | OmniError::StreamLifecycleIdempotencyConflict { .. } + | OmniError::Manifest(omnigraph::error::ManifestError { + kind: ManifestErrorKind::Conflict, + .. + }) => "graph stream authority changed; reread graph stream status and retry", + OmniError::RecoveryRequired { .. } => { + "graph recovery must settle before stream control can continue" + } + _ => match kind { + OfflineStreamControlKind::Block => { + "graph stream-block control failed; reread graph stream status and retry" + } + OfflineStreamControlKind::DeadLetter => { + "graph stream dead-letter control failed; reread graph stream status and retry" + } + }, + } +} + +fn project_logical_declaration( + table_key: &str, +) -> omnigraph::error::Result { + let (kind, type_name) = if let Some(type_name) = table_key.strip_prefix("node:") { + ("node", type_name) + } else if let Some(type_name) = table_key.strip_prefix("edge:") { + ("edge", type_name) + } else { + return Err(omnigraph::error::OmniError::manifest_internal( + "graph stream projection found an invalid logical declaration key", + )); + }; + if type_name.is_empty() { + return Err(omnigraph::error::OmniError::manifest_internal( + "graph stream projection found an empty logical declaration name", + )); + } + Ok(StreamLogicalDeclarationOutput { + kind: kind.to_string(), + type_name: type_name.to_string(), + }) +} + +fn project_stream_block_page( + page: omnigraph::db::StreamDataBlockPage, +) -> omnigraph::error::Result { + let declaration = project_logical_declaration(&page.table_key)?; + if page + .entries + .iter() + .any(|entry| entry.table_key != page.table_key) + { + return Err(omnigraph::error::OmniError::manifest_internal( + "graph stream block projection crossed logical declarations", + )); + } + Ok(StreamBlockPageOutput { + declaration, + block_token: page.block_token, + lifecycle_revision: page.lifecycle_revision, + correction_view_digest: page.correction_view_digest, + entries: page + .entries + .into_iter() + .map(|entry| StreamBlockEntryOutput { + ordinal: entry.ordinal, + logical_key: entry.logical_key, + current_blocked_winner_stream_token: entry.current_blocked_winner_stream_token, + violation_code: entry.violation_code, + field_path_or_group: entry.field_path_or_group, + violation_instance_id: entry.violation_instance_id, + allowed_actions: entry.allowed_actions, + }) + .collect(), + next_cursor: page.next_cursor, + }) +} + +fn project_stream_dead_letter_entry( + entry: omnigraph::db::StreamDeadLetterEntry, +) -> omnigraph::error::Result { + Ok(StreamDeadLetterEntryOutput { + declaration: project_logical_declaration(&entry.table_key)?, + logical_id: entry.logical_id, + occurrence_token: entry.occurrence_token, + predecessor_token: entry.predecessor_token, + write_id: entry.write_id, + contributor_id: entry.contributor_id, + payload_digest: entry.payload_digest, + reason_code: entry.reason_code, + candidate_ordinal: entry.candidate_ordinal, + }) +} + +fn project_stream_dead_letter_page( + page: omnigraph::db::StreamDeadLetterPage, +) -> omnigraph::error::Result { + Ok(StreamDeadLetterPageOutput { + source_manifest_version: page.source_manifest_version, + source_profile_revision: page.source_profile_revision, + entries: page + .entries + .into_iter() + .map(project_stream_dead_letter_entry) + .collect::>()?, + next_cursor: page.next_cursor, + }) +} + +fn project_stream_dead_letter_payload_page( + page: omnigraph::db::StreamDeadLetterPayloadPage, +) -> omnigraph::error::Result { + Ok(StreamDeadLetterPayloadPageOutput { + source_manifest_version: page.source_manifest_version, + source_profile_revision: page.source_profile_revision, + entries: page + .entries + .into_iter() + .map(|entry| { + Ok(StreamDeadLetterPayloadEntryOutput { + authority: project_stream_dead_letter_entry(entry.authority)?, + payload: entry.payload, + }) + }) + .collect::>()?, + next_cursor: page.next_cursor, + }) +} + pub async fn show_stream_data_block_config_dir( config_dir: impl AsRef, graph_id: impl AsRef, - table_key: impl AsRef, block_token: impl AsRef, cursor: Option<&str>, options: StreamBlockControlOptions, @@ -86,7 +231,6 @@ pub async fn show_stream_data_block_config_dir( } }; - let table_key = table_key.as_ref(); let block_token = block_token.as_ref(); let PreparedBlockCommand { config_dir, @@ -117,8 +261,10 @@ pub async fn show_stream_data_block_config_dir( let db = open_authorized_block_graph(&graph_id, &graph_uri, &desired, &backend, &state).await?; let authority = db.check_cluster_block_authority(guard).await?; - db.show_stream_data_block(authority, table_key, block_token, cursor) - .await + let page = db + .show_graph_stream_data_block(authority, block_token, cursor) + .await?; + project_stream_block_page(page) } .await; @@ -128,7 +274,7 @@ pub async fn show_stream_data_block_config_dir( diagnostics.push(Diagnostic::error( "stream_block_show_failed", format!("graph.{graph_id}"), - error.to_string(), + redacted_stream_control_error(OfflineStreamControlKind::Block, &error), )); None } @@ -148,7 +294,6 @@ pub async fn show_stream_data_block_config_dir( pub async fn correct_stream_data_block_config_dir( config_dir: impl AsRef, graph_id: impl AsRef, - table_key: impl AsRef, request: StreamDataCorrectionRequest, options: StreamBlockControlOptions, ) -> StreamBlockCorrectOutput { @@ -169,7 +314,6 @@ pub async fn correct_stream_data_block_config_dir( } }; - let table_key = table_key.as_ref(); let correction_id = request.correction_id.clone(); let PreparedBlockCommand { config_dir, @@ -200,8 +344,7 @@ pub async fn correct_stream_data_block_config_dir( let db = open_authorized_block_graph(&graph_id, &graph_uri, &desired, &backend, &state).await?; let authority = db.check_cluster_block_authority(guard).await?; - db.correct_stream_data_block(authority, table_key, request) - .await + db.correct_graph_stream_data_block(authority, request).await } .await; @@ -211,7 +354,7 @@ pub async fn correct_stream_data_block_config_dir( diagnostics.push(Diagnostic::error( "stream_block_correct_failed", format!("graph.{graph_id}"), - error.to_string(), + redacted_stream_control_error(OfflineStreamControlKind::Block, &error), )); None } @@ -287,7 +430,8 @@ pub async fn list_stream_dead_letters_config_dir( ) .await?; let authority = db.check_cluster_dead_letter_authority(guard).await?; - db.list_stream_dead_letters(authority, cursor).await + let page = db.list_stream_dead_letters(authority, cursor).await?; + project_stream_dead_letter_page(page) } .await; let page = match result { @@ -296,7 +440,7 @@ pub async fn list_stream_dead_letters_config_dir( diagnostics.push(Diagnostic::error( "stream_dead_letter_list_failed", format!("graph.{graph_id}"), - error.to_string(), + redacted_stream_control_error(OfflineStreamControlKind::DeadLetter, &error), )); None } @@ -372,8 +516,10 @@ pub async fn export_stream_dead_letters_config_dir( ) .await?; let authority = db.check_cluster_dead_letter_authority(guard).await?; - db.export_stream_dead_letter_payloads(authority, cursor) - .await + let page = db + .export_stream_dead_letter_payloads(authority, cursor) + .await?; + project_stream_dead_letter_payload_page(page) } .await; let page = match result { @@ -382,7 +528,7 @@ pub async fn export_stream_dead_letters_config_dir( diagnostics.push(Diagnostic::error( "stream_dead_letter_export_failed", format!("graph.{graph_id}"), - error.to_string(), + redacted_stream_control_error(OfflineStreamControlKind::DeadLetter, &error), )); None } @@ -747,3 +893,148 @@ async fn validated_dead_letter_offline_guard<'lock>( }, ) } + +#[cfg(test)] +mod projection_tests { + use super::*; + + fn raw_dead_letter_entry() -> omnigraph::db::StreamDeadLetterEntry { + omnigraph::db::StreamDeadLetterEntry { + stable_table_id: 41, + table_incarnation_id: 42, + table_key: "node:SecretDeclaration".to_string(), + logical_id: "logical-1".to_string(), + stream_incarnation_id: "secret-stream-incarnation".to_string(), + occurrence_token: "occurrence-1".to_string(), + predecessor_token: Some("predecessor-1".to_string()), + write_id: "write-1".to_string(), + contributor_id: "contributor-1".to_string(), + payload_digest: "sha256:payload".to_string(), + reason_code: "UNIQUE_VIOLATION".to_string(), + fold_operation_id: "secret-fold-operation".to_string(), + object_location: "s3://private-bucket/secret-object".to_string(), + object_digest: "sha256:secret-object".to_string(), + object_encoded_length: 99, + object_candidate_count: 3, + candidate_ordinal: 2, + } + } + + #[test] + fn block_projection_exposes_only_graph_level_correction_authority() { + let projected = project_stream_block_page(omnigraph::db::StreamDataBlockPage { + block_token: "opaque-block-token".to_string(), + stable_table_id: 41, + table_incarnation_id: 42, + table_key: "node:SecretDeclaration".to_string(), + lifecycle_revision: 7, + correction_view_digest: "sha256:view".to_string(), + entries: vec![omnigraph::db::StreamDataBlockEntry { + ordinal: 0, + table_key: "node:SecretDeclaration".to_string(), + logical_key: "logical-1".to_string(), + current_blocked_winner_stream_token: "winner-1".to_string(), + violation_code: "UNIQUE_VIOLATION".to_string(), + field_path_or_group: vec!["email".to_string()], + violation_instance_id: "violation-1".to_string(), + allowed_actions: vec!["WITHDRAW".to_string()], + }], + next_cursor: Some("opaque-cursor".to_string()), + }) + .unwrap(); + + let json = serde_json::to_string(&projected).unwrap(); + assert!(json.contains("opaque-block-token")); + assert!(json.contains("logical-1")); + assert!(json.contains(r#""kind":"node""#)); + assert!(json.contains(r#""type":"SecretDeclaration""#)); + for forbidden in [ + "stable_table_id", + "table_incarnation_id", + "table_key", + ] { + assert!( + !json.contains(forbidden), + "projection leaked {forbidden}: {json}" + ); + } + } + + #[test] + fn dead_letter_projections_hide_dataset_and_object_coordinates() { + let projected = project_stream_dead_letter_page(omnigraph::db::StreamDeadLetterPage { + source_manifest_version: 10, + source_profile_revision: 11, + token_table_version: 12, + token_transaction_uuid: "secret-token-transaction".to_string(), + entries: vec![raw_dead_letter_entry()], + next_cursor: Some("opaque-cursor".to_string()), + }) + .unwrap(); + let json = serde_json::to_string(&projected).unwrap(); + assert!(json.contains("logical-1")); + assert!(json.contains("source_manifest_version")); + assert!(json.contains(r#""kind":"node""#)); + assert!(json.contains(r#""type":"SecretDeclaration""#)); + for forbidden in [ + "stable_table_id", + "table_incarnation_id", + "table_key", + "stream_incarnation_id", + "fold_operation_id", + "object_location", + "object_digest", + "object_encoded_length", + "object_candidate_count", + "token_table_version", + "token_transaction_uuid", + "private-bucket", + "secret-stream-incarnation", + "secret-fold-operation", + "secret-token-transaction", + ] { + assert!( + !json.contains(forbidden), + "projection leaked {forbidden}: {json}" + ); + } + + let payload = + serde_json::value::RawValue::from_string(r#"{"nested":{"legal":true}}"#.to_string()) + .unwrap(); + let payload_projection = + project_stream_dead_letter_payload_page(omnigraph::db::StreamDeadLetterPayloadPage { + source_manifest_version: 10, + source_profile_revision: 11, + token_table_version: 12, + token_transaction_uuid: "secret-token-transaction".to_string(), + entries: vec![omnigraph::db::StreamDeadLetterPayloadEntry { + authority: raw_dead_letter_entry(), + payload, + }], + next_cursor: None, + }) + .unwrap(); + let payload_json = serde_json::to_string(&payload_projection).unwrap(); + assert!(payload_json.contains(r#""nested":{"legal":true}"#)); + assert!(!payload_json.contains("private-bucket")); + assert!(!payload_json.contains("token_transaction_uuid")); + } + + #[test] + fn diagnostics_do_not_render_physical_stream_identity() { + let error = omnigraph::error::OmniError::StreamLifecycleChanged { + stable_table_id: 41, + table_incarnation_id: 42, + expected_revision: 7, + current_revision: 8, + }; + let message = redacted_stream_control_error(OfflineStreamControlKind::Block, &error); + assert_eq!( + message, + "graph stream authority changed; reread graph stream status and retry" + ); + assert!(!message.contains("0000000000000029")); + assert!(!message.contains("000000000000002a")); + } +} diff --git a/crates/omnigraph-cluster/src/tests.rs b/crates/omnigraph-cluster/src/tests.rs index 19e0a85e..61eff24b 100644 --- a/crates/omnigraph-cluster/src/tests.rs +++ b/crates/omnigraph-cluster/src/tests.rs @@ -4493,7 +4493,6 @@ query seed($name: String, $age: I32) { let shown = show_stream_data_block_config_dir( dir.path(), "knowledge", - "node:Person", block_token, None, StreamBlockControlOptions { @@ -4563,6 +4562,59 @@ node Person { !live_after_schema_attempt.schema_source().contains("nickname"), "schema movement must wait until the disable continuation reaches DISABLED" ); + + let shown_page = shown.page.expect("the graph-level block page is present"); + let correction = omnigraph::db::StreamDataCorrectionRequest { + protocol_version: 1, + block_token: shown_page.block_token.clone(), + correction_id: "abababab-abab-4bab-8bab-abababababab".to_string(), + expected_lifecycle_revision: shown_page.lifecycle_revision, + actions: shown_page + .entries + .iter() + .map(|entry| omnigraph::db::StreamDataCorrectionAction::Withdraw { + ordinal: entry.ordinal, + logical_key: entry.logical_key.clone(), + current_blocked_winner_stream_token: entry + .current_blocked_winner_stream_token + .clone(), + }) + .collect(), + expected_plan_digest: None, + }; + let options = StreamBlockControlOptions { + actor: Some("stream-operator".to_string()), + confirm_stream_offline: true, + }; + let corrected = correct_stream_data_block_config_dir( + dir.path(), + "knowledge", + correction.clone(), + options.clone(), + ) + .await; + assert!(corrected.ok, "{corrected:?}"); + assert!(corrected.result.as_ref().is_some_and(|result| result.changed)); + + // The exact retry has no current block or caller-supplied table alias + // to resolve. It must discover the receipt across enrolled immutable + // identities before inspecting current lifecycle authority. + let replayed = correct_stream_data_block_config_dir( + dir.path(), + "knowledge", + correction, + options, + ) + .await; + assert!(replayed.ok, "{replayed:?}"); + assert!(replayed + .result + .as_ref() + .is_some_and(|result| !result.changed)); + assert_eq!( + corrected.result.as_ref().map(|result| &result.plan_digest), + replayed.result.as_ref().map(|result| &result.plan_digest) + ); } #[tokio::test] @@ -5407,7 +5459,6 @@ policies: {} let missing_authority = show_stream_data_block_config_dir( dir.path(), "knowledge", - "node:Person", "block-1", None, StreamBlockControlOptions::default(), @@ -5429,7 +5480,6 @@ policies: {} let unknown_graph = show_stream_data_block_config_dir( dir.path(), "other", - "node:Person", "block-1", None, StreamBlockControlOptions { @@ -5454,7 +5504,6 @@ policies: {} let unapplied_profile = show_stream_data_block_config_dir( dir.path(), "knowledge", - "node:Person", "block-1", None, StreamBlockControlOptions { @@ -5551,7 +5600,7 @@ policies: {} diagnostic.code == "stream_dead_letter_list_failed" && diagnostic .message - .contains("offline stream dead-letter control expected profile revision") + .contains("offline graph streaming authority is unavailable or changed") })); let stale_export = export_stream_dead_letters_config_dir( dir.path(), @@ -5565,7 +5614,7 @@ policies: {} diagnostic.code == "stream_dead_letter_export_failed" && diagnostic .message - .contains("offline stream dead-letter control expected profile revision") + .contains("offline graph streaming authority is unavailable or changed") })); } @@ -5682,7 +5731,6 @@ query arm_recovery($name: String, $age: I32) { let refused = correct_stream_data_block_config_dir( dir.path(), "knowledge", - "node:Person", request.clone(), StreamBlockControlOptions { actor: Some("stream-operator".to_string()), @@ -5710,7 +5758,6 @@ query arm_recovery($name: String, $age: I32) { let malformed = correct_stream_data_block_config_dir( dir.path(), "knowledge", - "node:Person", request, StreamBlockControlOptions { actor: Some("stream-operator".to_string()), @@ -5721,7 +5768,7 @@ query arm_recovery($name: String, $age: I32) { assert!(!malformed.ok, "{malformed:?}"); assert!(malformed.diagnostics.iter().any(|diagnostic| { diagnostic.code == "stream_block_correct_failed" - && diagnostic.message.contains("stream correction actions") + && diagnostic.message.contains("request is invalid") })); assert_eq!( graph_tree(&graph_root), diff --git a/crates/omnigraph-cluster/src/types.rs b/crates/omnigraph-cluster/src/types.rs index a3c96111..eed7fdbe 100644 --- a/crates/omnigraph-cluster/src/types.rs +++ b/crates/omnigraph-cluster/src/types.rs @@ -326,10 +326,47 @@ pub struct StreamBlockShowOutput { pub actor: Option, pub state_observations: StateObservations, #[serde(skip_serializing_if = "Option::is_none")] - pub page: Option, + pub page: Option, pub diagnostics: Vec, } +/// Graph-scoped projection of one correction-view entry. Physical table +/// identity remains an engine implementation detail. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct StreamBlockEntryOutput { + pub ordinal: u64, + pub logical_key: String, + pub current_blocked_winner_stream_token: String, + pub violation_code: String, + pub field_path_or_group: Vec, + pub violation_instance_id: String, + pub allowed_actions: Vec, +} + +/// Accepted-schema identity for one logical graph declaration. This is the +/// user-visible node/edge type, not a Lance dataset or manifest table key. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct StreamLogicalDeclarationOutput { + pub kind: String, + #[serde(rename = "type")] + pub type_name: String, +} + +/// Bounded graph-level correction page selected only by an opaque block token. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct StreamBlockPageOutput { + #[serde(flatten)] + pub declaration: StreamLogicalDeclarationOutput, + pub block_token: String, + pub lifecycle_revision: u64, + pub correction_view_digest: String, + pub entries: Vec, + pub next_cursor: Option, +} + #[derive(Debug, Clone, Serialize)] pub struct StreamBlockCorrectOutput { pub ok: bool, @@ -363,7 +400,7 @@ pub struct StreamDeadLetterListOutput { pub actor: Option, pub state_observations: StateObservations, #[serde(skip_serializing_if = "Option::is_none")] - pub page: Option, + pub page: Option, pub diagnostics: Vec, } @@ -378,10 +415,59 @@ pub struct StreamDeadLetterExportOutput { pub actor: Option, pub state_observations: StateObservations, #[serde(skip_serializing_if = "Option::is_none")] - pub page: Option, + pub page: Option, pub diagnostics: Vec, } +/// Graph-level terminal authority. Dataset coordinates, recovery ownership, +/// and immutable object descriptors are deliberately absent. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct StreamDeadLetterEntryOutput { + #[serde(flatten)] + pub declaration: StreamLogicalDeclarationOutput, + pub logical_id: String, + pub occurrence_token: String, + pub predecessor_token: Option, + pub write_id: String, + pub contributor_id: String, + pub payload_digest: String, + pub reason_code: String, + pub candidate_ordinal: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct StreamDeadLetterPageOutput { + pub source_manifest_version: u64, + pub source_profile_revision: u64, + pub entries: Vec, + pub next_cursor: Option, +} + +/// Descriptor-verified payload paired with its graph-level authority. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StreamDeadLetterPayloadEntryOutput { + pub authority: StreamDeadLetterEntryOutput, + pub payload: Box, +} + +impl PartialEq for StreamDeadLetterPayloadEntryOutput { + fn eq(&self, other: &Self) -> bool { + self.authority == other.authority && self.payload.get() == other.payload.get() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct StreamDeadLetterPayloadPageOutput { + pub source_manifest_version: u64, + pub source_profile_revision: u64, + pub entries: Vec, + pub next_cursor: Option, +} + /// Output of config-only `cluster apply`. "Applied" means recorded in the /// local cluster catalog (`__cluster/`); nothing applied here serves traffic — /// the server still boots from `omnigraph.yaml` until the server-boot stage. diff --git a/crates/omnigraph-server/src/handlers.rs b/crates/omnigraph-server/src/handlers.rs index 9dd65bb7..342282df 100644 --- a/crates/omnigraph-server/src/handlers.rs +++ b/crates/omnigraph-server/src/handlers.rs @@ -775,6 +775,147 @@ pub(crate) async fn server_stream_status( )) } +fn authorize_graph_stream_management<'a>( + handle: &GraphHandle, + actor: Option<&'a ResolvedActor>, +) -> std::result::Result<&'a str, ApiError> { + if handle.policy.is_none() { + // A configured graph policy is installed on the engine, which makes + // its graph-scoped StreamManage decision authoritative. Without one, + // preserve the server's open-mode / authenticated-default-deny + // contract before any management work starts. + authorize_request( + actor, + None, + PolicyRequest { + action: PolicyAction::StreamManage, + branch: None, + target_branch: None, + }, + )?; + } + Ok(actor.map_or("anonymous", |actor| actor.actor_id.as_ref())) +} + +async fn require_empty_graph_stream_management_body( + body: Body, +) -> std::result::Result<(), ApiError> { + // A zero-byte collection limit accepts a genuinely empty body while + // refusing on the first data byte, so selector-shaped payloads are rejected + // without buffering them or reaching authorization/engine effects. + axum::body::to_bytes(body, 0).await.map_err(|_| { + ApiError::bad_request("graph stream management does not accept a request body") + })?; + Ok(()) +} + +#[utoipa::path( + post, + path = "/stream/resume", + tag = "streaming", + operation_id = "stream_resume", + responses( + (status = 200, description = "All sealed streaming declarations were reopened through graph authority", body = StreamResumeOutput), + (status = 400, description = "The bodyless graph control received a request body", body = ErrorOutput), + (status = 401, description = "Unauthorized", body = ErrorOutput), + (status = 403, description = "The actor is not authorized for graph stream management", body = ErrorOutput), + (status = 409, description = "The graph streaming profile or lifecycle is not ready to resume", body = ErrorOutput), + (status = 413, description = "The bounded graph management operation exceeded a hard limit", body = ErrorOutput), + (status = 500, description = "Graph stream resume failed without exposing physical diagnostics", body = ErrorOutput), + (status = 503, description = "Graph recovery must complete before resume can proceed", body = ErrorOutput), + ), + security(("bearer_token" = [])), +)] +/// Reopen every sealed streaming declaration in the graph. +/// +/// The operation is deliberately bodyless and graph-wide: callers cannot +/// select a logical type, table, lane, dataset, or physical maintenance +/// target. The server supplies the bearer-resolved actor and the engine +/// performs the authoritative graph-scoped `stream_manage` check. +pub(crate) async fn server_stream_resume( + Extension(handle): Extension>, + actor: Option>, + body: Body, +) -> std::result::Result, ApiError> { + require_empty_graph_stream_management_body(body).await?; + let actor_id = + authorize_graph_stream_management(&handle, actor.as_ref().map(|Extension(actor)| actor))?; + let result = handle + .engine + .resume_served_graph_stream_as(actor_id) + .await + .map_err(|error| ApiError::from_graph_stream_management(error, "resume"))?; + Ok(Json(stream_resume_output(result))) +} + +#[utoipa::path( + post, + path = "/stream/maintenance/ensure-indices", + tag = "streaming", + operation_id = "stream_ensure_indices", + responses( + (status = 200, description = "Indexes were checked across the graph; affected enrolled declarations were sealed", body = StreamEnsureIndicesOutput), + (status = 400, description = "The bodyless graph control received a request body", body = ErrorOutput), + (status = 401, description = "Unauthorized", body = ErrorOutput), + (status = 403, description = "The actor is not authorized for graph stream management", body = ErrorOutput), + (status = 409, description = "The graph is not in the sealed maintenance posture", body = ErrorOutput), + (status = 413, description = "The bounded graph management operation exceeded a hard limit", body = ErrorOutput), + (status = 500, description = "Graph index maintenance failed without exposing physical diagnostics", body = ErrorOutput), + (status = 503, description = "Graph recovery must complete before maintenance can proceed", body = ErrorOutput), + ), + security(("bearer_token" = [])), +)] +/// Refresh graph index state; any enrolled declaration changed must be sealed. +pub(crate) async fn server_stream_ensure_indices( + Extension(handle): Extension>, + actor: Option>, + body: Body, +) -> std::result::Result, ApiError> { + require_empty_graph_stream_management_body(body).await?; + let actor_id = + authorize_graph_stream_management(&handle, actor.as_ref().map(|Extension(actor)| actor))?; + let result = handle + .engine + .ensure_served_graph_stream_indices_as(actor_id) + .await + .map_err(|error| ApiError::from_graph_stream_management(error, "ensure indices"))?; + Ok(Json(stream_ensure_indices_output(result))) +} + +#[utoipa::path( + post, + path = "/stream/maintenance/optimize", + tag = "streaming", + operation_id = "stream_optimize", + responses( + (status = 200, description = "The graph was considered for optimization; affected enrolled declarations were sealed", body = StreamOptimizeOutput), + (status = 400, description = "The bodyless graph control received a request body", body = ErrorOutput), + (status = 401, description = "Unauthorized", body = ErrorOutput), + (status = 403, description = "The actor is not authorized for graph stream management", body = ErrorOutput), + (status = 409, description = "The graph is not in the sealed maintenance posture", body = ErrorOutput), + (status = 413, description = "The bounded graph management operation exceeded a hard limit", body = ErrorOutput), + (status = 500, description = "Graph stream optimization failed without exposing physical diagnostics", body = ErrorOutput), + (status = 503, description = "Graph recovery must complete before maintenance can proceed", body = ErrorOutput), + ), + security(("bearer_token" = [])), +)] +/// Optimize the graph; any enrolled declaration changed must be sealed. +pub(crate) async fn server_stream_optimize( + Extension(handle): Extension>, + actor: Option>, + body: Body, +) -> std::result::Result, ApiError> { + require_empty_graph_stream_management_body(body).await?; + let actor_id = + authorize_graph_stream_management(&handle, actor.as_ref().map(|Extension(actor)| actor))?; + let result = handle + .engine + .optimize_served_graph_stream_as(actor_id) + .await + .map_err(|error| ApiError::from_graph_stream_management(error, "optimize"))?; + Ok(Json(stream_optimize_output(result))) +} + fn require_graph_stream_content_type( headers: &axum::http::HeaderMap, ) -> std::result::Result<(), ApiError> { diff --git a/crates/omnigraph-server/src/lib.rs b/crates/omnigraph-server/src/lib.rs index f7a511db..3ec2c34d 100644 --- a/crates/omnigraph-server/src/lib.rs +++ b/crates/omnigraph-server/src/lib.rs @@ -30,8 +30,10 @@ use api::{ CommitListQuery, ErrorCode, ErrorOutput, ExportRequest, GraphInfo, GraphListResponse, HealthOutput, IngestOutput, IngestRequest, InvokeStoredQueryRequest, InvokeStoredQueryResponse, QueriesCatalogOutput, QueryRequest, ReadOutput, ReadRequest, SchemaApplyOutput, - SchemaApplyRequest, SchemaOutput, SnapshotQuery, StreamIngestChallenge, StreamStatusOutput, - ingest_output, schema_apply_output, snapshot_payload, stream_status_output, + SchemaApplyRequest, SchemaOutput, SnapshotQuery, StreamEnsureIndicesOutput, + StreamIngestChallenge, StreamOptimizeOutput, StreamResumeOutput, StreamStatusOutput, + ingest_output, schema_apply_output, snapshot_payload, stream_ensure_indices_output, + stream_optimize_output, stream_resume_output, stream_status_output, }; pub use auth::{AWS_SECRET_ENV, EnvOrFileTokenSource, TokenSource, resolve_token_source}; use axum::body::{Body, Bytes}; @@ -110,6 +112,9 @@ fn hash_bearer_token(token: &str) -> BearerTokenHash { handlers::server_export, handlers::server_stream_status, handlers::server_stream_ingest, + handlers::server_stream_resume, + handlers::server_stream_ensure_indices, + handlers::server_stream_optimize, #[allow(deprecated)] handlers::server_change, handlers::server_mutate, handlers::server_list_queries, @@ -1179,6 +1184,62 @@ impl ApiError { _ => Self::internal("graph stream status failed"), } } + + /// Translate graph-wide lifecycle and maintenance failures without + /// exposing the private declaration, table, lane, dataset, block, or + /// recovery coordinates carried by trusted engine errors. + /// + /// This mapper is intentionally fail-closed: only the small stable set of + /// graph-actionable classes receives a public status. New engine variants + /// fall back to an opaque 500 until their disclosure posture is reviewed. + fn from_graph_stream_management(err: OmniError, operation: &'static str) -> Self { + match err { + OmniError::Policy(_) => { + Self::forbidden(format!("graph stream {operation} is forbidden")) + } + OmniError::ResourceLimitExceeded { + resource: _, + limit, + actual, + } => Self::resource_limit( + format!("graph stream {operation} limit exceeded: actual {actual}, limit {limit}"), + api::ResourceLimitOutput { + resource: "graph_stream_management".to_string(), + limit, + actual, + }, + ), + OmniError::AckUnknown { .. } + | OmniError::RecoveryRequired { .. } + | OmniError::StreamStatusBusy { .. } + | OmniError::StreamStatusChanged { .. } => Self::service_unavailable(format!( + "graph recovery must complete before graph stream {operation} can proceed" + )), + OmniError::Manifest(ref error) if matches!(error.kind, ManifestErrorKind::Conflict) => { + Self::conflict(format!("graph stream {operation} is not ready")) + } + OmniError::RetryableCommitConflict(_) + | OmniError::FoldRequired { .. } + | OmniError::StreamDataBlocked { .. } + | OmniError::StreamingDisablePending { .. } + | OmniError::StreamingRequiresClusterControlPlane + | OmniError::StreamingRequiresClusterRuntime { .. } + | OmniError::StreamingContentOperationUnsupported { .. } + | OmniError::StreamingAuthorityMismatch { .. } + | OmniError::StreamLifecycleChanged { .. } + | OmniError::StreamLifecycleIdempotencyConflict { .. } + | OmniError::StreamAuthorityRetired { .. } + | OmniError::StreamExportBlocked { .. } + | OmniError::StreamRetirementPlanChanged + | OmniError::StreamRetirementIdempotencyConflict { .. } + | OmniError::StreamBindingChanged { .. } + | OmniError::StreamSequenceConflict { .. } + | OmniError::StreamIdempotencyConflict { .. } => { + Self::conflict(format!("graph stream {operation} is not ready")) + } + _ => Self::internal(format!("graph stream {operation} failed")), + } + } } fn summarize_merge_conflicts(conflicts: &[api::MergeConflictOutput]) -> String { @@ -1500,6 +1561,87 @@ mod api_error_tests { assert_eq!(details.actual, 33); } + #[tokio::test] + async fn graph_stream_management_errors_redact_private_engine_evidence() { + let cases = vec![ + ( + OmniError::StreamLifecycleChanged { + stable_table_id: 41, + table_incarnation_id: 43, + expected_revision: 7, + current_revision: 9, + }, + StatusCode::CONFLICT, + "0000000000000029", + ), + ( + OmniError::StreamDataBlocked { + block_token: "private-block-token".to_string(), + }, + StatusCode::CONFLICT, + "private-block-token", + ), + ( + OmniError::RecoveryRequired { + operation_id: "private-recovery-operation".to_string(), + reason: "private recovery reason".to_string(), + }, + StatusCode::SERVICE_UNAVAILABLE, + "private-recovery-operation", + ), + ( + OmniError::Lance("s3://private-bucket/private-dataset".to_string()), + StatusCode::INTERNAL_SERVER_ERROR, + "private-bucket", + ), + ( + OmniError::Policy("private policy evaluator detail".to_string()), + StatusCode::FORBIDDEN, + "private policy evaluator detail", + ), + ]; + + for (engine_error, expected_status, private_evidence) in cases { + let response = + ApiError::from_graph_stream_management(engine_error, "resume").into_response(); + assert_eq!(response.status(), expected_status); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let error: ErrorOutput = serde_json::from_slice(&body).unwrap(); + assert!(!error.error.contains(private_evidence), "{}", error.error); + assert!(error.merge_conflicts.is_empty()); + assert!(error.manifest_conflict.is_none()); + assert!(error.read_set_conflict.is_none()); + assert!(error.key_conflict.is_none()); + assert!(error.resource_limit.is_none()); + assert!(error.recovery_required.is_none()); + } + + let response = ApiError::from_graph_stream_management( + OmniError::ResourceLimitExceeded { + resource: "private table inventory".to_string(), + limit: 32, + actual: 33, + }, + "ensure indices", + ) + .into_response(); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let error: ErrorOutput = serde_json::from_slice(&body).unwrap(); + assert!(!error.error.contains("private table inventory")); + let details = error + .resource_limit + .expect("public graph-management resource limit"); + assert_eq!(details.resource, "graph_stream_management"); + assert_eq!(details.limit, 32); + assert_eq!(details.actual, 33); + assert!(error.recovery_required.is_none()); + } + #[tokio::test] async fn stream_management_conflicts_serialize_as_409() { let cases = [ @@ -1613,6 +1755,12 @@ pub fn build_app(state: AppState) -> Router { .route("/export", post(server_export)) .route("/stream/status", get(server_stream_status)) .route("/stream/ingest", post(server_stream_ingest)) + .route("/stream/resume", post(server_stream_resume)) + .route( + "/stream/maintenance/ensure-indices", + post(server_stream_ensure_indices), + ) + .route("/stream/maintenance/optimize", post(server_stream_optimize)) // /read and /change are kept indefinitely for back-compat; // their handlers carry #[deprecated] so the OpenAPI operation is // flagged and their responses include RFC 9745 Deprecation + diff --git a/crates/omnigraph-server/tests/auth_policy.rs b/crates/omnigraph-server/tests/auth_policy.rs index 5cb8f6e9..20f3230a 100644 --- a/crates/omnigraph-server/tests/auth_policy.rs +++ b/crates/omnigraph-server/tests/auth_policy.rs @@ -82,6 +82,29 @@ async fn protected_routes_require_bearer_token() { .unwrap(); assert_eq!(status_response.status(), StatusCode::UNAUTHORIZED); + for path in [ + "/stream/resume", + "/stream/maintenance/ensure-indices", + "/stream/maintenance/optimize", + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .uri(g(path)) + .method(Method::POST) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "protected graph streaming control {path}" + ); + } + let stream_body_polled = Arc::new(AtomicBool::new(false)); let body_probe = Arc::clone(&stream_body_polled); let response = app diff --git a/crates/omnigraph-server/tests/multi_graph.rs b/crates/omnigraph-server/tests/multi_graph.rs index 0bc504e7..9e3734a9 100644 --- a/crates/omnigraph-server/tests/multi_graph.rs +++ b/crates/omnigraph-server/tests/multi_graph.rs @@ -14,8 +14,9 @@ use omnigraph::db::Omnigraph; use omnigraph::loader::{LoadMode, load_jsonl}; use omnigraph_server::api::{ ChangeRequest, ErrorOutput, ExportRequest, QueryRequest, ReadRequest, StreamDriverStateOutput, - StreamIngestChallenge, StreamIngestKindOutput, StreamIngestLineOutput, - StreamIngestStatusOutput, StreamProfileModeOutput, StreamStatusOutput, + StreamEnsureIndicesOutput, StreamIngestChallenge, StreamIngestKindOutput, + StreamIngestLineOutput, StreamIngestStatusOutput, StreamOptimizeOutput, + StreamProfileModeOutput, StreamResumeOutput, StreamStatusOutput, }; use omnigraph_server::{AppState, build_app}; use serde_json::Value; @@ -622,6 +623,98 @@ async fn cluster_boot_installs_enabled_stream_runtime_authority() { assert_eq!(status.driver.state, StreamDriverStateOutput::Running); assert!(!status.rebuild.ready); + for path in [ + "/graphs/knowledge/stream/resume", + "/graphs/knowledge/stream/maintenance/ensure-indices", + "/graphs/knowledge/stream/maintenance/optimize", + ] { + let (rejected_status, rejected_body) = json_response( + &app, + Request::builder() + .method(Method::POST) + .uri(path) + .header("content-type", "application/json") + .body(Body::from(r#"{"type":"Person"}"#)) + .unwrap(), + ) + .await; + assert_eq!( + rejected_status, + StatusCode::BAD_REQUEST, + "selector body must be rejected by {path}: {rejected_body}" + ); + let error: ErrorOutput = serde_json::from_value(rejected_body).unwrap(); + assert!( + error.error.contains("does not accept a request body"), + "{path}: {error:?}" + ); + assert!( + engine.stream_status().await.unwrap().tables.is_empty(), + "a rejected graph control body cannot initialize a private declaration" + ); + } + + let (resume_status, resume_body) = json_response( + &app, + Request::builder() + .method(Method::POST) + .uri("/graphs/knowledge/stream/resume") + .body(Body::empty()) + .unwrap(), + ) + .await; + assert_eq!(resume_status, StatusCode::OK, "{resume_body}"); + let resume: StreamResumeOutput = serde_json::from_value(resume_body.clone()).unwrap(); + assert_eq!(resume.profile_revision, status.profile_revision); + assert_eq!(resume.enrolled_declarations, 0); + assert_eq!(resume.resumed_declarations, 0); + assert_eq!(resume.already_open_declarations, 0); + + let (ensure_status, ensure_body) = json_response( + &app, + Request::builder() + .method(Method::POST) + .uri("/graphs/knowledge/stream/maintenance/ensure-indices") + .body(Body::empty()) + .unwrap(), + ) + .await; + assert_eq!(ensure_status, StatusCode::OK, "{ensure_body}"); + let _: StreamEnsureIndicesOutput = serde_json::from_value(ensure_body.clone()).unwrap(); + + let (optimize_status, optimize_body) = json_response( + &app, + Request::builder() + .method(Method::POST) + .uri("/graphs/knowledge/stream/maintenance/optimize") + .body(Body::empty()) + .unwrap(), + ) + .await; + assert_eq!(optimize_status, StatusCode::OK, "{optimize_body}"); + let _: StreamOptimizeOutput = serde_json::from_value(optimize_body.clone()).unwrap(); + + for output in [&resume_body, &ensure_body, &optimize_body] { + let rendered = output.to_string(); + for forbidden in [ + "type_name", + "table_key", + "stable_table_id", + "table_incarnation_id", + "stream_incarnation_id", + "lane", + "dataset", + "operation_id", + "block_token", + "recovery", + ] { + assert!( + !rendered.contains(forbidden), + "graph streaming control leaked '{forbidden}': {output}" + ); + } + } + let (status, body) = json_response( &app, Request::builder() @@ -994,7 +1087,7 @@ query streamed_edge() { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] -async fn authenticated_default_deny_refuses_stream_ingest_before_body_ownership() { +async fn authenticated_default_deny_refuses_stream_writes_before_effects() { let (_temp, state, engine) = enabled_stream_state(vec![( "stream-actor".to_string(), "stream-secret".to_string(), @@ -1028,6 +1121,32 @@ async fn authenticated_default_deny_refuses_stream_ingest_before_body_ownership( engine.stream_status().await.unwrap().tables.is_empty(), "a default-denied request cannot lazily enroll a private lane" ); + for path in [ + "/graphs/knowledge/stream/resume", + "/graphs/knowledge/stream/maintenance/ensure-indices", + "/graphs/knowledge/stream/maintenance/optimize", + ] { + let (status, body) = json_response( + &app, + Request::builder() + .method(Method::POST) + .uri(path) + .header("authorization", "Bearer stream-secret") + .body(Body::empty()) + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN, "{path}: {body}"); + let error: ErrorOutput = serde_json::from_value(body).unwrap(); + assert!( + error.error.contains("default-deny mode"), + "{path}: {error:?}" + ); + assert!( + engine.stream_status().await.unwrap().tables.is_empty(), + "default-denied graph control {path} cannot create a private lane" + ); + } let (status, body) = json_response( &app, Request::builder() diff --git a/crates/omnigraph-server/tests/openapi.rs b/crates/omnigraph-server/tests/openapi.rs index ef9c022d..05539fb3 100644 --- a/crates/omnigraph-server/tests/openapi.rs +++ b/crates/omnigraph-server/tests/openapi.rs @@ -298,6 +298,40 @@ fn graph_stream_ingest_documents_ndjson_and_token_preconditions() { assert!(responses["428"]["headers"]["Cache-Control"].is_object()); } +#[test] +fn graph_stream_controls_are_bodyless_graph_wide_operations() { + let doc = openapi_json(); + for (path, schema) in [ + ("/graphs/{graph_id}/stream/resume", "StreamResumeOutput"), + ( + "/graphs/{graph_id}/stream/maintenance/ensure-indices", + "StreamEnsureIndicesOutput", + ), + ( + "/graphs/{graph_id}/stream/maintenance/optimize", + "StreamOptimizeOutput", + ), + ] { + let operation = &doc["paths"][path]["post"]; + assert!(operation.is_object(), "missing graph stream control {path}"); + assert!( + operation.get("requestBody").is_none(), + "graph stream control {path} must not accept a selector or actor body" + ); + assert_eq!( + operation["responses"]["200"]["content"]["application/json"]["schema"]["$ref"], + format!("#/components/schemas/{schema}") + ); + for status in ["400", "401", "403", "409", "413", "500", "503"] { + assert_eq!( + operation["responses"][status]["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/ErrorOutput", + "graph stream control {path} response {status}" + ); + } + } +} + #[test] fn stream_aware_export_documents_pre_header_failures() { let doc = openapi_json(); @@ -487,6 +521,7 @@ const EXPECTED_SCHEMAS: &[&str] = &[ "StreamDriverErrorOutput", "StreamDriverStateOutput", "StreamDriverStatusOutput", + "StreamEnsureIndicesOutput", "StreamIngestChallenge", "StreamIngestKindOutput", "StreamIngestLineOutput", @@ -494,10 +529,12 @@ const EXPECTED_SCHEMAS: &[&str] = &[ "StreamIngestStatusOutput", "StreamLastFoldStatusOutput", "StreamLifecycleOutput", + "StreamOptimizeOutput", "StreamPendingStatusOutput", "StreamProfileModeOutput", "StreamRebuildBlockerOutput", "StreamRebuildStatusOutput", + "StreamResumeOutput", "StreamStatusOutput", "StreamStrictBlockStatusOutput", "StreamTokenCountsOutput", diff --git a/crates/omnigraph/src/db/manifest.rs b/crates/omnigraph/src/db/manifest.rs index 777ab907..1cd4a767 100644 --- a/crates/omnigraph/src/db/manifest.rs +++ b/crates/omnigraph/src/db/manifest.rs @@ -129,9 +129,10 @@ pub(crate) use stream_profile::{ }; pub(crate) use stream_token::stream_token_authority_plan_digest; pub(crate) use token_store::{ - LifecycleLedgerRecord, StreamTokenAuthorityEntry, lookup_authority_retirement_receipt, - lookup_claim_receipt, lookup_lifecycle_ledger_record_by_id, lookup_management_receipt, - lookup_profile_management_receipt, lookup_stream_correction_receipt, + LifecycleLedgerRecord, MAX_LIFECYCLE_LEDGER_LOOKUP_KEYS_PER_SCAN, StreamTokenAuthorityEntry, + lookup_authority_retirement_receipt, lookup_claim_receipt, + lookup_lifecycle_ledger_record_by_id, lookup_lifecycle_ledger_records_batched, + lookup_management_receipt, lookup_profile_management_receipt, lookup_stream_correction_receipt, open_stream_token_authority_at, open_stream_token_authority_head, stage_authority_retirement_receipt, stage_lifecycle_ledger_records, stage_management_receipt, stage_profile_management_receipt, stage_stream_correction_effect, diff --git a/crates/omnigraph/src/db/manifest/stream.rs b/crates/omnigraph/src/db/manifest/stream.rs index c966cfbc..87ff4fef 100644 --- a/crates/omnigraph/src/db/manifest/stream.rs +++ b/crates/omnigraph/src/db/manifest/stream.rs @@ -2858,6 +2858,38 @@ pub(crate) fn stream_disable_drain_id( Ok(ShardId::from_bytes(bytes).to_string()) } +/// Derive the stable UUID-v4 occurrence for graph-level resume of one exact +/// lifecycle revision. +/// +/// The graph coordinator deliberately persists no parallel work queue. A +/// replacement request reconstructs the same per-lane occurrence from the +/// immutable table lifetime and its exact SEALED compare token, while a later +/// SEALED revision receives a different occurrence. UUID-v4 shape is retained +/// because recovery-v15's request grammar is frozen. +pub(crate) fn graph_stream_resume_id( + identity: TableIdentity, + lifecycle_revision: u64, +) -> Result { + const DOMAIN: &[u8] = b"omnigraph.graph-stream-resume-id.v1\0"; + identity.validate()?; + if lifecycle_revision == 0 { + return Err(OmniError::manifest_internal( + "graph stream resume requires a positive lifecycle revision", + )); + } + let mut hasher = Sha256::new(); + hash_bytes(&mut hasher, DOMAIN); + hash_bytes(&mut hasher, &identity.stable_table_id.to_be_bytes()); + hash_bytes(&mut hasher, &identity.table_incarnation_id.to_be_bytes()); + hash_bytes(&mut hasher, &lifecycle_revision.to_be_bytes()); + let digest = hasher.finalize(); + let mut bytes = [0_u8; 16]; + bytes.copy_from_slice(&digest[..16]); + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + Ok(ShardId::from_bytes(bytes).to_string()) +} + /// Canonical request selected by metadata-only disable-drain adoption. pub(crate) fn stream_disable_drain_adoption_request_payload( disable_operation_id: &str, @@ -5346,6 +5378,22 @@ mod tests { assert_eq!(parsed.to_string(), operation_id); } + #[test] + fn graph_resume_derives_stable_revision_scoped_uuid_v4_occurrences() { + let identity = TableIdentity::new(7, 9).unwrap(); + let first = graph_stream_resume_id(identity, 3).unwrap(); + assert_eq!(graph_stream_resume_id(identity, 3).unwrap(), first); + assert_ne!(graph_stream_resume_id(identity, 4).unwrap(), first); + assert_ne!( + graph_stream_resume_id(TableIdentity::new(7, 10).unwrap(), 3).unwrap(), + first + ); + let parsed = ShardId::parse_str(&first).unwrap(); + assert_eq!(parsed.get_version_num(), 4); + assert_eq!(parsed.to_string(), first); + assert!(graph_stream_resume_id(identity, 0).is_err()); + } + fn entry() -> StreamLifecycleEntry { let shard_id = "22222222-2222-4222-8222-222222222222".to_string(); let binding = StreamPhysicalBinding { diff --git a/crates/omnigraph/src/db/manifest/token_store.rs b/crates/omnigraph/src/db/manifest/token_store.rs index c0398915..5a277ca8 100644 --- a/crates/omnigraph/src/db/manifest/token_store.rs +++ b/crates/omnigraph/src/db/manifest/token_store.rs @@ -64,6 +64,10 @@ const MAX_LIFECYCLE_LEDGER_RECORD_JSON_BYTES: usize = 16 * 1024; const MAX_LIFECYCLE_LEDGER_TRANSACTION_JSON_BYTES: usize = MAX_LIFECYCLE_LEDGER_RECORDS_PER_TRANSACTION * MAX_LIFECYCLE_LEDGER_RECORD_JSON_BYTES; const MAX_LIFECYCLE_LEDGER_TRANSACTION_ARROW_BYTES: u64 = 1024 * 1024; +/// One graph-level receipt probe normally covers every enrolled declaration in +/// one structured scan. Larger graphs split into explicit bounded chunks so +/// the IN predicate and retained decoded receipts cannot grow without limit. +pub(crate) const MAX_LIFECYCLE_LEDGER_LOOKUP_KEYS_PER_SCAN: usize = 256; #[derive(Debug, Clone, PartialEq, Eq)] struct LifecycleLedgerEnvelope { @@ -1573,6 +1577,108 @@ pub(crate) async fn lookup_lifecycle_ledger_record( .transpose() } +fn collect_requested_lifecycle_ledger_records( + requested: &BTreeMap, + envelopes: Vec, +) -> Result> { + if requested.is_empty() || requested.len() > MAX_LIFECYCLE_LEDGER_LOOKUP_KEYS_PER_SCAN { + return Err(OmniError::resource_limit( + "stream_lifecycle_ledger_lookup_keys", + MAX_LIFECYCLE_LEDGER_LOOKUP_KEYS_PER_SCAN as u64, + u64::try_from(requested.len()).unwrap_or(u64::MAX), + )); + } + for (lookup_key, tag) in requested { + for (name, value) in [("record tag", *tag), ("record lookup key", lookup_key.as_str())] { + if value.is_empty() || value.trim() != value { + return Err(OmniError::manifest_internal(format!( + "lifecycle ledger {name} must be non-empty canonical text" + ))); + } + } + } + if envelopes.len() > requested.len() { + return Err(OmniError::manifest_internal( + "lifecycle ledger batched lookup returned more rows than requested keys", + )); + } + + let mut selected = BTreeMap::new(); + for envelope in envelopes { + let expected_tag = requested.get(&envelope.record_lookup_key).ok_or_else(|| { + OmniError::manifest_internal( + "lifecycle ledger batched lookup returned an unexpected record lookup key", + ) + })?; + if envelope.record_tag != *expected_tag { + return Err(OmniError::manifest_internal(format!( + "lifecycle ledger batched lookup expected trusted record tag '{expected_tag}' but received '{}'", + envelope.record_tag + ))); + } + let lookup_key = envelope.record_lookup_key.clone(); + let record = LifecycleLedgerRecord::from_envelope(envelope)?; + if selected.insert(lookup_key, record).is_some() { + return Err(OmniError::manifest_internal( + "lifecycle ledger batched lookup returned a duplicate record lookup key", + )); + } + } + Ok(selected) +} + +/// Resolve one bounded set of immutable lifecycle-ledger keys with one Lance +/// scan. The predicate deliberately selects by lookup key alone: a row with an +/// unexpected tag is returned and refused instead of being hidden by a tag +/// filter. The rows+1 limit and cross-batch map catch duplicate unenforced keys. +pub(crate) async fn lookup_lifecycle_ledger_records_batched( + dataset: &Dataset, + authority: &StreamTokenAuthorityEntry, + requested: &BTreeMap, +) -> Result> { + validate_exact_dataset(dataset, authority).await?; + // Validate the bound and request grammar before constructing the IN list. + collect_requested_lifecycle_ledger_records(requested, Vec::new())?; + + let mut scanner = dataset.scan(); + scanner.filter_expr(col("record_lookup_key").in_list( + requested.keys().cloned().map(lit).collect(), + false, + )); + scanner.batch_size(MAX_LIFECYCLE_LEDGER_RECORDS_PER_TRANSACTION); + scanner.batch_size_bytes(MAX_LIFECYCLE_LEDGER_TRANSACTION_ARROW_BYTES); + scanner + .limit( + Some( + i64::try_from(requested.len().saturating_add(1)).map_err(|_| { + OmniError::manifest_internal( + "lifecycle ledger batched lookup row limit exceeds i64", + ) + })?, + ), + None, + ) + .map_err(|error| OmniError::Lance(error.to_string()))?; + let mut stream = scanner + .try_into_stream() + .await + .map_err(|error| OmniError::Lance(error.to_string()))?; + let mut envelopes = Vec::new(); + while let Some(batch) = stream + .try_next() + .await + .map_err(|error| OmniError::Lance(error.to_string()))? + { + envelopes.extend(lifecycle_ledger_envelopes_from_batch(&batch, None)?); + if envelopes.len() > requested.len() { + return Err(OmniError::manifest_internal( + "lifecycle ledger batched lookup returned more rows than requested keys", + )); + } + } + collect_requested_lifecycle_ledger_records(requested, envelopes) +} + /// Resolve one manifest-selected immutable ledger head by its exact record ID. /// /// Lifecycle rows retain the selected record ID rather than the operation @@ -3136,6 +3242,50 @@ mod tests { assert!(!tags.contains(PROFILE_MANAGEMENT_RECEIPT_TAG)); } + #[test] + fn batched_lifecycle_lookup_refuses_duplicate_unexpected_and_unbounded_rows() { + let records = typed_lifecycle_records(); + let first = records[0].to_envelope().unwrap(); + let second = records[1].to_envelope().unwrap(); + let third = records[2].to_envelope().unwrap(); + let requested = BTreeMap::from([ + (first.record_lookup_key.clone(), records[0].record_tag()), + (second.record_lookup_key.clone(), records[1].record_tag()), + ]); + + let duplicate = collect_requested_lifecycle_ledger_records( + &requested, + vec![first.clone(), first], + ) + .expect_err("one unenforced lookup key may not resolve twice"); + assert!(duplicate.to_string().contains("duplicate record lookup key")); + + let unexpected = collect_requested_lifecycle_ledger_records(&requested, vec![third]) + .expect_err("the structured scan may not return an unrequested key"); + assert!(unexpected.to_string().contains("unexpected record lookup key")); + + let too_many = (0..=MAX_LIFECYCLE_LEDGER_LOOKUP_KEYS_PER_SCAN) + .map(|ordinal| { + ( + format!("stream-control-test:{ordinal}"), + MANAGEMENT_RECEIPT_TAG, + ) + }) + .collect(); + let error = collect_requested_lifecycle_ledger_records(&too_many, Vec::new()) + .expect_err("the one-scan request has an explicit key bound"); + assert!(matches!( + error, + OmniError::ResourceLimitExceeded { + resource, + limit, + actual, + } if resource == "stream_lifecycle_ledger_lookup_keys" + && limit == MAX_LIFECYCLE_LEDGER_LOOKUP_KEYS_PER_SCAN as u64 + && actual == limit + 1 + )); + } + #[test] fn token_plan_bounds_fail_loudly_for_arrow_and_recovery_json() { let row = authority_row(); diff --git a/crates/omnigraph/src/db/mod.rs b/crates/omnigraph/src/db/mod.rs index 52a9b7fa..81ec551b 100644 --- a/crates/omnigraph/src/db/mod.rs +++ b/crates/omnigraph/src/db/mod.rs @@ -28,9 +28,10 @@ pub use omnigraph::{ pub use omnigraph::{ GraphStreamChunkSource, GraphStreamDeclaration, GraphStreamDeclarationStatus, GraphStreamDrainStatus, GraphStreamDriverErrorStatus, GraphStreamDriverStatus, - GraphStreamIngestHandle, GraphStreamIngestStart, GraphStreamLastFoldStatus, - GraphStreamOperationalStatus, GraphStreamPendingStatus, GraphStreamRebuildBlocker, - GraphStreamRebuildStatus, GraphStreamStrictBlockStatus, GraphStreamTokenCounts, + GraphStreamEnsureIndicesResult, GraphStreamIngestHandle, GraphStreamIngestStart, + GraphStreamLastFoldStatus, GraphStreamOperationalStatus, GraphStreamOptimizeResult, + GraphStreamPendingStatus, GraphStreamRebuildBlocker, GraphStreamRebuildStatus, + GraphStreamResumeResult, GraphStreamStrictBlockStatus, GraphStreamTokenCounts, }; pub(crate) use omnigraph::{ DeferredTableFork, StreamAuthorityRetirementExportProvenance, WriteAuthorityToken, WriteTxn, diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index 8b7e2529..b955fbd0 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -45,6 +45,7 @@ mod stream_enrollment; mod stream_graph_ingest; mod stream_ingest; pub(crate) mod stream_lifecycle; +mod stream_management; mod stream_ndjson; mod stream_profile; mod stream_rebind; @@ -74,6 +75,10 @@ pub use stream_dead_letter::{ StreamDeadLetterEncodingCostForTest, failpoint_measure_stream_dead_letter_object_for_test, }; #[doc(hidden)] +pub use stream_management::{ + GraphStreamEnsureIndicesResult, GraphStreamOptimizeResult, GraphStreamResumeResult, +}; +#[doc(hidden)] pub use stream_ndjson::{GraphStreamChunkSource, GraphStreamIngestHandle, GraphStreamIngestStart}; #[doc(hidden)] pub use stream_profile::{ @@ -2678,7 +2683,9 @@ impl Omnigraph { &self, actor_id: &str, ) -> Result> { - table_ops::ensure_indices_sealed_as(self, actor_id).await + Ok(table_ops::ensure_indices_sealed_as(self, actor_id) + .await? + .pending) } #[cfg(feature = "failpoints")] diff --git a/crates/omnigraph/src/db/omnigraph/stream_correction.rs b/crates/omnigraph/src/db/omnigraph/stream_correction.rs index b5174db8..d5f4a818 100644 --- a/crates/omnigraph/src/db/omnigraph/stream_correction.rs +++ b/crates/omnigraph/src/db/omnigraph/stream_correction.rs @@ -27,11 +27,12 @@ use super::stream_lifecycle::{ }; use super::{CheckedClusterBlockAuthority, Omnigraph}; use crate::db::manifest::stream::{ - ManagementReceipt, STREAM_CORRECTION_OPERATION_KIND, - STREAM_DATA_BLOCK_VALIDATION_CONTRACT_VERSION, StreamCorrectionReceipt, - StreamCorrectionReceiptPreimage, StreamDataCorrectionOutcome, StreamLifecycle, StrictBlock, - StrictBlockEvidence, build_data_block_correction_successor, stream_correction_result_payload, - stream_graph_identity_digest, stream_lifecycle_authority_digest, + MANAGEMENT_RECEIPT_TAG, ManagementReceipt, STREAM_CORRECTION_OPERATION_KIND, + STREAM_CORRECTION_RECEIPT_TAG, STREAM_DATA_BLOCK_VALIDATION_CONTRACT_VERSION, + StreamCorrectionReceipt, StreamCorrectionReceiptPreimage, StreamDataCorrectionOutcome, + StreamLifecycle, StrictBlock, StrictBlockEvidence, build_data_block_correction_successor, + stream_correction_result_payload, stream_graph_identity_digest, + stream_lifecycle_authority_digest, }; use crate::db::manifest::stream_token::{ PayloadDigest, PayloadDigestInput, StreamFoldAttributionSummary, StreamTokenAuthorityRow, @@ -39,9 +40,11 @@ use crate::db::manifest::stream_token::{ stream_fold_attribution_commitment, }; use crate::db::manifest::{ - RecoveryAuthorityToken, RecoveryLineageIntent, RecoveryStreamFoldCut, SidecarTablePin, + LifecycleLedgerRecord, MAX_LIFECYCLE_LEDGER_LOOKUP_KEYS_PER_SCAN, RecoveryAuthorityToken, + RecoveryLineageIntent, RecoveryStreamFoldCut, SidecarTablePin, complete_stream_correction_sidecar_v20, confirm_stream_correction_sidecar_v20, - lookup_management_receipt, lookup_stream_correction_receipt, new_stream_correction_sidecar_v20, + lookup_lifecycle_ledger_records_batched, lookup_management_receipt, + lookup_stream_correction_receipt, new_stream_correction_sidecar_v20, open_stream_token_authority_head, stage_stream_correction_effect, stream_token_authority_entry_for_dataset, stream_token_authority_plan_digest, validate_stream_token_plan_bounds, write_sidecar, @@ -338,14 +341,15 @@ fn validate_request_uuid(field: &str, value: &str) -> Result<()> { } impl Omnigraph { - /// Reconstruct one page from the exact retained generation named by a - /// durable `DataBlock`. No digest is trusted as a substitute for reopening - /// the cut and rerunning the validator. + /// Reconstruct one page for the graph's exact current `DataBlock` token. + /// + /// The token is the only public selector. Its current immutable table + /// lifetime is resolved from manifest authority and is never supplied by + /// the caller. #[doc(hidden)] - pub async fn show_stream_data_block( + pub async fn show_graph_stream_data_block( &self, authority: CheckedClusterBlockAuthority<'_>, - table_key: &str, block_token: &str, cursor: Option<&str>, ) -> Result { @@ -355,7 +359,17 @@ impl Omnigraph { .to_string(), }); } - self.show_stream_data_block_inner(Some(&authority), table_key, block_token, cursor) + let table_key = { + let _profile_guard = self.write_queue().acquire_stream_profile_shared().await; + let snapshot = self.open_write_txn(None).await?; + validate_optional_block_authority(Some(&authority), snapshot.base.stream_profile())?; + self.resolve_current_data_block_table_key( + &snapshot.base, + block_token, + "stream block show", + )? + }; + self.show_stream_data_block_inner(Some(&authority), &table_key, block_token, cursor) .await } @@ -430,14 +444,14 @@ impl Omnigraph { Ok(page) } - /// Apply one exact, bounded correction plan. Receipt lookup intentionally - /// precedes current block/revision checks so a response lost after the - /// terminal manifest CAS remains an ordinary idempotent retry. + /// Apply one graph-selected correction plan. Receipt lookup spans every + /// currently enrolled immutable table identity before current-block + /// resolution, so an exact retry remains possible after the block cleared + /// or the manifest moved. A caller never selects a physical stream lane. #[doc(hidden)] - pub async fn correct_stream_data_block( + pub async fn correct_graph_stream_data_block( &self, authority: CheckedClusterBlockAuthority<'_>, - table_key: &str, request: StreamDataCorrectionRequest, ) -> Result { request.validate_shape()?; @@ -448,10 +462,209 @@ impl Omnigraph { }); } let actor = authority.actor().to_string(); - self.correct_stream_data_block_inner(Some(&authority), table_key, request, &actor) + + // Shape and external authority checks intentionally precede recovery. + // Once admitted, recovery may settle an exact prior attempt before the + // receipt-first graph lookup classifies this invocation. + self.heal_pending_recovery_sidecars_outcome().await?; + let resolution = { + let _profile_guard = self.write_queue().acquire_stream_profile_shared().await; + let txn = self.open_write_txn(None).await?; + validate_optional_block_authority(Some(&authority), txn.base.stream_profile())?; + if let Some((identity, result)) = self + .correction_receipt_result_across_enrolled_streams(&txn, &actor, &request) + .await? + { + self.notify_stream_fold_pressure(identity); + return Ok(result); + } + self.resolve_current_data_block_table_key( + &txn.base, + &request.block_token, + "stream block correct", + )? + }; + + self.correct_stream_data_block_inner(Some(&authority), &resolution, request, &actor) .await } + async fn correction_receipt_result_across_enrolled_streams( + &self, + txn: &super::WriteTxn, + actor: &str, + request: &StreamDataCorrectionRequest, + ) -> Result< + Option<( + crate::db::manifest::TableIdentity, + StreamDataCorrectionResult, + )>, + > { + let graph_identity_digest = + stream_graph_identity_digest(&txn.authority.schema_identity_domain)?; + let token_dataset = txn.base.open_stream_token_authority().await?; + let mut selected = None; + let lanes = self.ordered_enrolled_stream_lanes(&txn.base)?; + let identities_per_scan = MAX_LIFECYCLE_LEDGER_LOOKUP_KEYS_PER_SCAN / 2; + if identities_per_scan == 0 { + return Err(OmniError::manifest_internal( + "lifecycle ledger batched lookup cannot fit one correction receipt pair", + )); + } + + for lane_chunk in lanes.chunks(identities_per_scan) { + let mut requested = BTreeMap::new(); + let mut candidates = Vec::with_capacity(lane_chunk.len()); + for lane in lane_chunk { + let lifecycle = txn.base.stream_lifecycle(lane.identity).ok_or_else(|| { + OmniError::manifest_internal( + "ordered enrolled stream lane has no lifecycle authority", + ) + })?; + let stream_incarnation_id = + lifecycle.enrollment_receipt.stream_incarnation_id.clone(); + let correction_lookup_key = StreamCorrectionReceipt::lookup_key_for( + &graph_identity_digest, + lane.identity, + &stream_incarnation_id, + &request.block_token, + &request.correction_id, + )?; + let management_lookup_key = ManagementReceipt::lookup_key_for( + &graph_identity_digest, + lane.identity, + &stream_incarnation_id, + STREAM_CORRECTION_OPERATION_KIND, + &request.correction_id, + )?; + if requested + .insert(correction_lookup_key.clone(), STREAM_CORRECTION_RECEIPT_TAG) + .is_some() + || requested + .insert(management_lookup_key.clone(), MANAGEMENT_RECEIPT_TAG) + .is_some() + { + return Err(OmniError::manifest_internal( + "graph correction receipt lookup derived a duplicate exact key", + )); + } + candidates.push(( + lane.identity, + lane.table_key.clone(), + stream_incarnation_id, + correction_lookup_key, + management_lookup_key, + )); + } + let mut records = lookup_lifecycle_ledger_records_batched( + &token_dataset, + txn.base.stream_token_authority(), + &requested, + ) + .await?; + + for ( + identity, + table_key, + stream_incarnation_id, + correction_lookup_key, + management_lookup_key, + ) in candidates + { + let correction_receipt = match records.remove(&correction_lookup_key) { + Some(LifecycleLedgerRecord::StreamCorrectionReceipt(receipt)) => Some(receipt), + Some(_) => { + return Err(OmniError::manifest_internal( + "graph correction lookup decoded another receipt family", + )); + } + None => None, + }; + let management_receipt = match records.remove(&management_lookup_key) { + Some(LifecycleLedgerRecord::ManagementReceipt(receipt)) => Some(receipt), + Some(_) => { + return Err(OmniError::manifest_internal( + "graph correction lookup decoded another management family", + )); + } + None => None, + }; + let (receipt, management) = match (correction_receipt, management_receipt) { + (None, None) => continue, + (None, Some(_)) | (Some(_), None) => { + return Err(correction_idempotency_conflict(identity, request)); + } + (Some(receipt), Some(management)) => (receipt, management), + }; + validate_correction_receipt_pair(&receipt, &management)?; + let plan_digest = normalized_request_plan_digest( + &txn.catalog, + identity, + &txn.authority.schema_ir_hash, + &stream_incarnation_id, + &table_key, + actor, + request, + )?; + validate_correction_receipt_retry(&receipt, actor, &plan_digest, request)?; + let result = StreamDataCorrectionResult { + changed: false, + correction_id: receipt.correction_id, + plan_digest: receipt.correction_plan_digest, + graph_commit_id: receipt.graph_commit_id, + lifecycle_revision: receipt.resulting_lifecycle_revision, + manifest_version: receipt.resulting_manifest_version, + }; + if selected.replace((identity, result)).is_some() { + return Err(OmniError::manifest_internal( + "one graph correction id resolved to more than one enrolled stream identity", + )); + } + } + if !records.is_empty() { + return Err(OmniError::manifest_internal( + "graph correction receipt lookup left an unclassified exact record", + )); + } + } + Ok(selected) + } + + fn resolve_current_data_block_table_key( + &self, + snapshot: &crate::db::manifest::Snapshot, + block_token: &str, + operation: &str, + ) -> Result { + let mut selected = None; + for lane in self.ordered_enrolled_stream_lanes(snapshot)? { + let Some(lifecycle) = snapshot.stream_lifecycle(lane.identity) else { + continue; + }; + let Some(block) = lifecycle.strict_block.as_ref() else { + continue; + }; + if block.block_token != block_token { + continue; + } + if !matches!(block.evidence, StrictBlockEvidence::DataBlock { .. }) { + return Err(OmniError::manifest_conflict( + "the selected stream authority block is not data-correctable", + )); + } + if selected.replace(lane.table_key).is_some() { + return Err(OmniError::manifest_internal( + "one stream block token resolved to more than one enrolled stream identity", + )); + } + } + selected.ok_or_else(|| { + OmniError::manifest_not_found(format!( + "{operation} cannot resolve the supplied graph stream block token" + )) + }) + } + async fn correct_stream_data_block_inner( &self, authority: Option<&CheckedClusterBlockAuthority<'_>>, diff --git a/crates/omnigraph/src/db/omnigraph/stream_driver.rs b/crates/omnigraph/src/db/omnigraph/stream_driver.rs index e996f589..938ca4b1 100644 --- a/crates/omnigraph/src/db/omnigraph/stream_driver.rs +++ b/crates/omnigraph/src/db/omnigraph/stream_driver.rs @@ -1251,6 +1251,25 @@ impl Omnigraph { .await } + /// Release a foreign resident before graph-level lifecycle control claims + /// the bounded profile's sole root slot. + /// + /// Unlike logical route settlement, this deliberately does not require a + /// disposition from the target lane itself: resume calls it while that + /// target is still SEALED, and a stale pre-seal driver trigger must not be + /// mistaken for a failed resume. The subsequent recovery-v15 adapter and + /// its exact post-open handoff own the target's disposition. + pub(super) async fn release_foreign_stream_resident_for_graph_control( + self: &Arc, + target_identity: TableIdentity, + ) -> Result<()> { + maybe_fail(names::STREAM_DRIVER_BEFORE_ROUND_ACQUIRE)?; + let _round_admission = self.stream_workers.acquire_stream_fold_round().await; + self.seed_initial_stream_discovery_if_needed().await?; + self.release_foreign_stream_resident_under_graph_fence(target_identity) + .await + } + async fn settle_stream_lane_for_graph( self: &Arc, expected_identity: TableIdentity, diff --git a/crates/omnigraph/src/db/omnigraph/stream_management.rs b/crates/omnigraph/src/db/omnigraph/stream_management.rs new file mode 100644 index 00000000..b419d965 --- /dev/null +++ b/crates/omnigraph/src/db/omnigraph/stream_management.rs @@ -0,0 +1,200 @@ +//! Graph-wide served stream controls. +//! +//! This is a thin orchestration layer over the existing per-lane lifecycle and +//! graph-wide SEALED-maintenance adapters. The manifest remains the work list; +//! no second coordinator, durable queue, or recovery vocabulary is introduced. + +use std::sync::Arc; + +use crate::db::manifest::stream::graph_stream_resume_id; +use crate::db::manifest::{StreamLifecycle, StreamProfileMode}; +use crate::error::{OmniError, Result}; + +use super::*; + +/// Graph-redacted result of converging every enrolled SEALED declaration to +/// `OPEN`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[doc(hidden)] +pub struct GraphStreamResumeResult { + pub profile_revision: u64, + pub enrolled_declarations: u64, + pub resumed_declarations: u64, + pub already_open_declarations: u64, +} + +/// Graph-redacted result of one SEALED EnsureIndices pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[doc(hidden)] +pub struct GraphStreamEnsureIndicesResult { + pub changed: bool, + pub pending_index_count: u64, +} + +/// Graph-redacted result of one SEALED Optimize pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[doc(hidden)] +pub struct GraphStreamOptimizeResult { + pub changed: bool, + pub pending_index_count: u64, + pub requires_repair: bool, +} + +fn graph_resume_serial_queue_key() -> crate::db::write_queue::TableQueueKey { + // Real table aliases are always node:/edge:-prefixed. This process-local + // key serializes only graph resume orchestration; it is not durable + // authority and is never interpreted as a table identity. + ("__graph_stream_resume__".to_string(), None) +} + +fn count_u64(field: &str, count: usize) -> Result { + u64::try_from(count).map_err(|_| { + OmniError::manifest_internal(format!( + "graph stream {field} count exceeds the u64 contract" + )) + }) +} + +impl Omnigraph { + /// Resume every currently enrolled SEALED declaration in deterministic + /// graph order. OPEN declarations are idempotent no-ops; a DRAINING lane or + /// unresolved strict block refuses the whole preflight before any resume + /// effect starts. + #[doc(hidden)] + pub async fn resume_served_graph_stream_as( + self: &Arc, + actor_id: &str, + ) -> Result { + // The policy decision belongs to the graph operation, including the + // zero-lane case. Per-lane resume is deliberately crate-private and + // cannot be used as a public selector-shaped authorization surface. + self.enforce( + omnigraph_policy::PolicyAction::StreamManage, + &omnigraph_policy::ResourceScope::Graph, + Some(actor_id), + )?; + + let _resume_serial = self + .write_queue() + .acquire(&graph_resume_serial_queue_key()) + .await; + + // Capture one immutable graph work list while the exact checked + // ENABLED delegation cannot transition. Release this read gate before + // invoking lane resume: that adapter acquires the root resident-worker + // opportunity before the profile gate, and preserving that order is + // required for shutdown freedom from lock inversion. + let profile_guard = self.write_queue().acquire_stream_profile_shared().await; + self.ensure_streaming_ingest_runtime_authorized().await?; + let frozen = self.open_write_txn(None).await?; + if frozen.base.stream_profile().mode() != StreamProfileMode::Enabled { + return Err(OmniError::StreamingAuthorityMismatch { + reason: "graph stream resume requires the exact ENABLED profile".to_string(), + }); + } + let profile_revision = frozen.base.stream_profile().profile_revision; + let lanes = self.ordered_enrolled_stream_lanes(&frozen.base)?; + let mut sealed = Vec::new(); + let mut already_open = 0_usize; + for lane in &lanes { + let lifecycle = frozen.base.stream_lifecycle(lane.identity).ok_or_else(|| { + OmniError::StreamingAuthorityMismatch { + reason: "graph stream enrollment changed during resume preflight".to_string(), + } + })?; + if lifecycle.strict_block.is_some() { + return Err(OmniError::manifest_conflict( + "graph stream resume is blocked by unresolved validation evidence", + )); + } + match lifecycle.lifecycle { + StreamLifecycle::Open => already_open += 1, + StreamLifecycle::Sealed => sealed.push(( + lane.table_key.clone(), + lane.identity, + lifecycle.lifecycle_revision, + )), + StreamLifecycle::Draining => { + return Err(OmniError::manifest_conflict( + "graph stream resume requires every drain to reach SEALED first", + )); + } + } + } + drop(profile_guard); + + let mut resumed = 0_usize; + for (table_key, identity, lifecycle_revision) in sealed { + let resume_id = graph_stream_resume_id(identity, lifecycle_revision)?; + // A retry may encounter the empty OPEN owner installed by an + // earlier lane before its caller observed the handoff. Reuse the + // graph route coordinator to release any such foreign resident + // without asking the still-SEALED target for an OPEN disposition. + self.release_foreign_stream_resident_for_graph_control(identity) + .await?; + self.stream_resume_as( + &table_key, + &resume_id, + lifecycle_revision, + crate::db::manifest::stream::StreamResumeMode::ResumeSealed, + actor_id, + ) + .await?; + // Resume must install its fresh empty writer before publishing the + // OPEN receipt. Settle that exact owner through the existing finite + // driver round before selecting the next lane, otherwise several + // graph declarations would contend for the bounded profile's sole + // resident slot. This is a handoff, not a second lifecycle effect. + self.fold_resident_stream_lane_for_handoff(identity, &table_key) + .await?; + resumed += 1; + } + + Ok(GraphStreamResumeResult { + profile_revision, + enrolled_declarations: count_u64("enrolled declaration", lanes.len())?, + resumed_declarations: count_u64("resumed declaration", resumed)?, + already_open_declarations: count_u64("already-open declaration", already_open)?, + }) + } + + /// Run the existing one-publication SEALED EnsureIndices adapter and + /// project away every physical table detail. + #[doc(hidden)] + pub async fn ensure_served_graph_stream_indices_as( + &self, + actor_id: &str, + ) -> Result { + let outcome = table_ops::ensure_indices_sealed_as(self, actor_id).await?; + Ok(GraphStreamEnsureIndicesResult { + changed: outcome.changed, + pending_index_count: count_u64("pending index", outcome.pending.len())?, + }) + } + + /// Run the existing one-publication SEALED Optimize adapter and project + /// away table aliases, versions, fragment counts, and physical paths. + #[doc(hidden)] + pub async fn optimize_served_graph_stream_as( + &self, + actor_id: &str, + ) -> Result { + let stats = optimize::optimize_all_tables_sealed_as(self, actor_id).await?; + let pending_index_count = stats.iter().try_fold(0_u64, |total, stat| { + total + .checked_add(count_u64("pending index", stat.pending_indexes.len())?) + .ok_or_else(|| { + OmniError::manifest_internal( + "graph stream pending index count exceeds the u64 contract", + ) + }) + })?; + Ok(GraphStreamOptimizeResult { + changed: stats.iter().any(|stat| stat.committed), + pending_index_count, + requires_repair: stats + .iter() + .any(|stat| stat.skipped == Some(SkipReason::DriftNeedsRepair)), + }) + } +} diff --git a/crates/omnigraph/src/db/omnigraph/stream_retirement.rs b/crates/omnigraph/src/db/omnigraph/stream_retirement.rs index 9071e36a..84cc4ec1 100644 --- a/crates/omnigraph/src/db/omnigraph/stream_retirement.rs +++ b/crates/omnigraph/src/db/omnigraph/stream_retirement.rs @@ -71,6 +71,9 @@ const DEAD_LETTER_PAGE_ENVELOPE_BYTES: usize = 64 * 1024; const DEAD_LETTER_CURSOR_MAX_DECODED_BYTES: usize = 4 * 1024; const DEAD_LETTER_CURSOR_MAX_ENCODED_BYTES: usize = DEAD_LETTER_CURSOR_MAX_DECODED_BYTES.div_ceil(3) * 4; +const DEAD_LETTER_CURSOR_FORMAT: &str = "omnigraph.stream-dead-letter-cursor.v1"; +const DEAD_LETTER_CURSOR_BINDING_DOMAIN: &[u8] = + b"omnigraph.stream-dead-letter-cursor-binding.v1\0"; const MAX_STREAM_TERMINAL_TOKEN_SAMPLE_ROWS: usize = 16; /// Complete in-process full-root cut envelope shared by authority retirement @@ -215,13 +218,18 @@ struct CapturedStreamDeadLetterPage { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct StreamDeadLetterCursor { - source_manifest_version: u64, - source_profile_revision: u64, - token_table_version: u64, - token_transaction_uuid: String, - last_stable_table_id: u64, - last_table_incarnation_id: u64, - last_logical_id: String, + format: String, + declaration_kind: StreamDeadLetterCursorDeclarationKind, + declaration_type: String, + logical_id: String, + authority_binding: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum StreamDeadLetterCursorDeclarationKind { + Node, + Edge, } /// Exact frozen branch member named by one retired export. The receipt binds @@ -522,15 +530,14 @@ impl Omnigraph { .capture_stream_dead_letter_page(&authority, cursor) .await?; let cut_cursor = |entry: &CapturedStreamDeadLetterEntry| { - encode_stream_dead_letter_cursor(&StreamDeadLetterCursor { - source_manifest_version: captured.source_manifest_version, - source_profile_revision: captured.source_profile_revision, - token_table_version: captured.token_table_version, - token_transaction_uuid: captured.token_transaction_uuid.clone(), - last_stable_table_id: entry.key.identity.stable_table_id, - last_table_incarnation_id: entry.key.identity.table_incarnation_id, - last_logical_id: entry.key.logical_id.clone(), - }) + encode_stream_dead_letter_cursor( + captured.source_manifest_version, + captured.source_profile_revision, + captured.token_table_version, + &captured.token_transaction_uuid, + &entry.key, + &entry.public.table_key, + ) }; let mut entries = Vec::new(); @@ -642,24 +649,17 @@ impl Omnigraph { let cursor = encoded_cursor .map(decode_stream_dead_letter_cursor) .transpose()?; - if cursor.as_ref().is_some_and(|cursor| { - cursor.source_manifest_version != snapshot.version() - || cursor.source_profile_revision != snapshot.stream_profile().profile_revision - || cursor.token_table_version != token_head.table_version - || cursor.token_transaction_uuid != token_head.transaction_uuid - }) { - return Err(OmniError::StreamingAuthorityMismatch { - reason: "stream dead-letter cursor belongs to another manifest/profile/token cut" - .to_string(), - }); - } - let after = cursor.as_ref().map(|cursor| StreamDeadLetterKey { - identity: TableIdentity { - stable_table_id: cursor.last_stable_table_id, - table_incarnation_id: cursor.last_table_incarnation_id, - }, - logical_id: cursor.last_logical_id.clone(), - }); + let after = cursor + .as_ref() + .map(|cursor| { + resolve_stream_dead_letter_cursor( + cursor, + &snapshot, + token_head.table_version, + &token_head.transaction_uuid, + ) + }) + .transpose()?; let table_keys = snapshot .entries() @@ -798,15 +798,14 @@ impl Omnigraph { "dead-letter scan found additional entries but retained no page boundary", ) })?; - Some(encode_stream_dead_letter_cursor(&StreamDeadLetterCursor { - source_manifest_version: snapshot.version(), - source_profile_revision: snapshot.stream_profile().profile_revision, - token_table_version: token_head.table_version, - token_transaction_uuid: token_head.transaction_uuid.clone(), - last_stable_table_id: last.key.identity.stable_table_id, - last_table_incarnation_id: last.key.identity.table_incarnation_id, - last_logical_id: last.key.logical_id.clone(), - })?) + Some(encode_stream_dead_letter_cursor( + snapshot.version(), + snapshot.stream_profile().profile_revision, + token_head.table_version, + &token_head.transaction_uuid, + &last.key, + &last.public.table_key, + )?) } else { None }; @@ -1809,8 +1808,100 @@ fn validate_dead_letter_payload_binding( Ok(()) } -fn encode_stream_dead_letter_cursor(cursor: &StreamDeadLetterCursor) -> Result { - let bytes = serde_json::to_vec(cursor).map_err(|error| { +impl StreamDeadLetterCursorDeclarationKind { + fn as_str(self) -> &'static str { + match self { + Self::Node => "node", + Self::Edge => "edge", + } + } + + fn table_key(self, type_name: &str) -> String { + format!("{}:{type_name}", self.as_str()) + } +} + +fn stream_dead_letter_cursor_declaration( + table_key: &str, +) -> Result<(StreamDeadLetterCursorDeclarationKind, String)> { + let (kind, type_name) = if let Some(type_name) = table_key.strip_prefix("node:") { + (StreamDeadLetterCursorDeclarationKind::Node, type_name) + } else if let Some(type_name) = table_key.strip_prefix("edge:") { + (StreamDeadLetterCursorDeclarationKind::Edge, type_name) + } else { + return Err(OmniError::manifest_internal( + "stream dead-letter cursor cannot project an invalid logical declaration key", + )); + }; + if type_name.is_empty() { + return Err(OmniError::manifest_internal( + "stream dead-letter cursor cannot project an empty logical declaration name", + )); + } + Ok((kind, type_name.to_string())) +} + +fn stream_dead_letter_cursor_authority_binding( + cursor: &StreamDeadLetterCursor, + source_manifest_version: u64, + source_profile_revision: u64, + token_table_version: u64, + token_transaction_uuid: &str, + identity: TableIdentity, +) -> String { + // The manifest-selected token transaction UUID is high-entropy authority + // that is deliberately absent from the public cursor. Derive a private + // cut key first, then frame that key on both sides of the public cursor + // preimage. This avoids a forgeable checksum (and secret-prefix length + // extension) without introducing process-local state or another persisted + // signing key. + let mut cut_key_hasher = Sha256::new(); + hash_field(&mut cut_key_hasher, DEAD_LETTER_CURSOR_BINDING_DOMAIN); + hash_u64(&mut cut_key_hasher, source_manifest_version); + hash_u64(&mut cut_key_hasher, source_profile_revision); + hash_u64(&mut cut_key_hasher, token_table_version); + hash_field(&mut cut_key_hasher, token_transaction_uuid.as_bytes()); + hash_u64(&mut cut_key_hasher, identity.stable_table_id); + hash_u64(&mut cut_key_hasher, identity.table_incarnation_id); + let cut_key = cut_key_hasher.finalize(); + + let mut hasher = Sha256::new(); + hash_field(&mut hasher, DEAD_LETTER_CURSOR_BINDING_DOMAIN); + hash_field(&mut hasher, &cut_key); + hash_field(&mut hasher, cursor.format.as_bytes()); + hash_field(&mut hasher, cursor.declaration_kind.as_str().as_bytes()); + hash_field(&mut hasher, cursor.declaration_type.as_bytes()); + hash_field(&mut hasher, cursor.logical_id.as_bytes()); + hash_field(&mut hasher, &cut_key); + finish_digest(hasher) +} + +#[allow(clippy::too_many_arguments)] +fn encode_stream_dead_letter_cursor( + source_manifest_version: u64, + source_profile_revision: u64, + token_table_version: u64, + token_transaction_uuid: &str, + key: &StreamDeadLetterKey, + table_key: &str, +) -> Result { + let (declaration_kind, declaration_type) = stream_dead_letter_cursor_declaration(table_key)?; + let mut cursor = StreamDeadLetterCursor { + format: DEAD_LETTER_CURSOR_FORMAT.to_string(), + declaration_kind, + declaration_type, + logical_id: key.logical_id.clone(), + authority_binding: String::new(), + }; + cursor.authority_binding = stream_dead_letter_cursor_authority_binding( + &cursor, + source_manifest_version, + source_profile_revision, + token_table_version, + token_transaction_uuid, + key.identity, + ); + let bytes = serde_json::to_vec(&cursor).map_err(|error| { OmniError::manifest_internal(format!( "failed to encode stream dead-letter cursor: {error}" )) @@ -1852,20 +1943,51 @@ fn decode_stream_dead_letter_cursor(encoded: &str) -> Result Result { + let table_key = cursor.declaration_kind.table_key(&cursor.declaration_type); + let entry = + snapshot + .entry(&table_key) + .ok_or_else(|| OmniError::StreamingAuthorityMismatch { + reason: "stream dead-letter cursor belongs to another graph authority cut" + .to_string(), + })?; + let expected = stream_dead_letter_cursor_authority_binding( + cursor, + snapshot.version(), + snapshot.stream_profile().profile_revision, + token_table_version, + token_transaction_uuid, + entry.identity, + ); + if cursor.authority_binding != expected { + return Err(OmniError::StreamingAuthorityMismatch { + reason: "stream dead-letter cursor belongs to another graph authority cut".to_string(), + }); + } + Ok(StreamDeadLetterKey { + identity: entry.identity, + logical_id: cursor.logical_id.clone(), + }) +} + fn enforce_dead_letter_page_bound(resource: &'static str, page: &impl Serialize) -> Result<()> { let encoded = serde_json::to_vec(page).map_err(|error| { OmniError::manifest_internal(format!("failed to size stream dead-letter page: {error}")) @@ -2397,25 +2519,87 @@ fn finish_digest(hasher: Sha256) -> String { mod tests { use super::*; - fn cursor() -> StreamDeadLetterCursor { - StreamDeadLetterCursor { - source_manifest_version: 9, - source_profile_revision: 4, - token_table_version: 7, - token_transaction_uuid: "00000000-0000-4000-8000-000000000001".to_string(), - last_stable_table_id: 11, - last_table_incarnation_id: 12, - last_logical_id: "person-42".to_string(), + const CURSOR_TOKEN_TRANSACTION: &str = "00000000-0000-4000-8000-000000000001"; + + fn cursor_key() -> StreamDeadLetterKey { + StreamDeadLetterKey { + identity: TableIdentity::new(11, 12).unwrap(), + logical_id: "person-42".to_string(), } } #[test] - fn dead_letter_cursor_round_trips_exact_cut_and_boundary() { - let expected = cursor(); - let encoded = encode_stream_dead_letter_cursor(&expected).unwrap(); + fn dead_letter_cursor_exposes_only_logical_boundary_and_bound_digest() { + let encoded = encode_stream_dead_letter_cursor( + 9, + 4, + 7, + CURSOR_TOKEN_TRANSACTION, + &cursor_key(), + "node:Person", + ) + .unwrap(); + let decoded_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(&encoded) + .unwrap(); + let decoded_json: serde_json::Value = serde_json::from_slice(&decoded_bytes).unwrap(); + assert_eq!(decoded_json["format"], DEAD_LETTER_CURSOR_FORMAT); + assert_eq!(decoded_json["declaration_kind"], "node"); + assert_eq!(decoded_json["declaration_type"], "Person"); + assert_eq!(decoded_json["logical_id"], "person-42"); + assert!( + decoded_json["authority_binding"] + .as_str() + .is_some_and(|binding| binding.starts_with("sha256:")) + ); + let rendered = String::from_utf8(decoded_bytes).unwrap(); + for forbidden in [ + "source_manifest_version", + "source_profile_revision", + "token_table_version", + "token_transaction_uuid", + "stable_table_id", + "table_incarnation_id", + CURSOR_TOKEN_TRANSACTION, + ] { + assert!( + !rendered.contains(forbidden), + "cursor leaked {forbidden}: {rendered}" + ); + } + + let decoded = decode_stream_dead_letter_cursor(&encoded).unwrap(); + assert_eq!( + decoded.declaration_kind, + StreamDeadLetterCursorDeclarationKind::Node + ); + assert_eq!(decoded.declaration_type, "Person"); + assert_eq!(decoded.logical_id, "person-42"); assert_eq!( - decode_stream_dead_letter_cursor(&encoded).unwrap(), - expected + decoded.authority_binding, + stream_dead_letter_cursor_authority_binding( + &decoded, + 9, + 4, + 7, + CURSOR_TOKEN_TRANSACTION, + cursor_key().identity, + ) + ); + + let mut tampered = decoded; + tampered.logical_id = "person-43".to_string(); + assert_ne!( + tampered.authority_binding, + stream_dead_letter_cursor_authority_binding( + &tampered, + 9, + 4, + 7, + CURSOR_TOKEN_TRANSACTION, + cursor_key().identity, + ), + "the logical page boundary must be integrity-bound to the hidden cut" ); } diff --git a/crates/omnigraph/src/db/omnigraph/table_ops.rs b/crates/omnigraph/src/db/omnigraph/table_ops.rs index 1530c374..14618366 100644 --- a/crates/omnigraph/src/db/omnigraph/table_ops.rs +++ b/crates/omnigraph/src/db/omnigraph/table_ops.rs @@ -9,6 +9,16 @@ enum EnsureIndicesMode<'a> { }, } +pub(super) struct SealedEnsureIndicesOutcome { + pub(super) changed: bool, + pub(super) pending: Vec, +} + +struct EnsureIndicesOutcome { + changed: bool, + pending: Vec, +} + impl<'a> EnsureIndicesMode<'a> { fn actor_id(self) -> Option<&'a str> { match self { @@ -49,19 +59,27 @@ pub(super) async fn ensure_indices(db: &Omnigraph) -> Result> .await .current_branch() .map(str::to_string); - ensure_indices_for_branch(db, current_branch.as_deref(), EnsureIndicesMode::Ambient).await + Ok( + ensure_indices_for_branch(db, current_branch.as_deref(), EnsureIndicesMode::Ambient) + .await? + .pending, + ) } pub(super) async fn ensure_indices_on(db: &Omnigraph, branch: &str) -> Result> { let branch = normalize_branch_name(branch)?; - ensure_indices_for_branch(db, branch.as_deref(), EnsureIndicesMode::Ambient).await + Ok( + ensure_indices_for_branch(db, branch.as_deref(), EnsureIndicesMode::Ambient) + .await? + .pending, + ) } #[cfg_attr(not(feature = "failpoints"), allow(dead_code))] pub(super) async fn ensure_indices_sealed_as( db: &Omnigraph, actor_id: &str, -) -> Result> { +) -> Result { db.enforce( omnigraph_policy::PolicyAction::StreamManage, &omnigraph_policy::ResourceScope::Graph, @@ -69,7 +87,13 @@ pub(super) async fn ensure_indices_sealed_as( )?; db.ensure_streaming_sealed_maintenance_runtime_authorized() .await?; - ensure_indices_for_branch(db, None, EnsureIndicesMode::SealedMaintenance { actor_id }).await + let outcome = + ensure_indices_for_branch(db, None, EnsureIndicesMode::SealedMaintenance { actor_id }) + .await?; + Ok(SealedEnsureIndicesOutcome { + changed: outcome.changed, + pending: outcome.pending, + }) } #[cfg(feature = "failpoints")] @@ -120,7 +144,7 @@ async fn ensure_indices_for_branch( db: &Omnigraph, branch: Option<&str>, mode: EnsureIndicesMode<'_>, -) -> Result> { +) -> Result { if mode.is_sealed_maintenance() && branch.is_some() { return Err(OmniError::manifest( "SEALED stream maintenance is canonical-main only", @@ -311,6 +335,7 @@ async fn ensure_indices_for_branch( ) }) .collect::>(); + let changed = !recovery_pins.is_empty(); if mode.is_sealed_maintenance() { // Recheck under the retained profile gate and immediately before the // lower writer domains so stale checked runtime cannot authorize the @@ -761,7 +786,7 @@ async fn ensure_indices_for_branch( pending.append(&mut table_pending); } } - Ok(pending) + Ok(EnsureIndicesOutcome { changed, pending }) } fn pre_minted_index_transaction( diff --git a/crates/omnigraph/tests/forbidden_apis.rs b/crates/omnigraph/tests/forbidden_apis.rs index 92b7b995..8586bf43 100644 --- a/crates/omnigraph/tests/forbidden_apis.rs +++ b/crates/omnigraph/tests/forbidden_apis.rs @@ -209,6 +209,10 @@ const STREAM_LIFECYCLE_V14: WriteProtocol = WriteProtocol::Exact("private firehose lifecycle recovery v14"); const STREAM_LIFECYCLE_V14_V15: WriteProtocol = WriteProtocol::Composed("private firehose lifecycle recovery v14 + resume recovery v15"); +const STREAM_SEALED_ENSURE_INDICES_V16: WriteProtocol = + WriteProtocol::Exact("sealed EnsureIndices recovery v16"); +const STREAM_SEALED_OPTIMIZE_V17: WriteProtocol = + WriteProtocol::Bounded("sealed Optimize recovery v17"); const STREAM_REBIND_V18: WriteProtocol = WriteProtocol::Exact("private physical rebind recovery v18"); const STREAM_RETIREMENT_V19: WriteProtocol = @@ -288,7 +292,7 @@ write_surfaces! { "failpoint_withdraw_stream_token_for_retirement_test", ], "db/omnigraph/stream_correction.rs" => STREAM_CORRECTION_V20 => [ - "correct_stream_data_block", + "correct_graph_stream_data_block", ], "db/omnigraph/stream_correction.rs" => WriteProtocol::TestOnly => [ "failpoint_correct_stream_data_block_for_test", @@ -310,6 +314,15 @@ write_surfaces! { "start_stream_fold_driver", "shutdown_stream_fold_driver", ], + "db/omnigraph/stream_management.rs" => STREAM_LIFECYCLE_V14_V15 => [ + "resume_served_graph_stream_as", + ], + "db/omnigraph/stream_management.rs" => STREAM_SEALED_ENSURE_INDICES_V16 => [ + "ensure_served_graph_stream_indices_as", + ], + "db/omnigraph/stream_management.rs" => STREAM_SEALED_OPTIMIZE_V17 => [ + "optimize_served_graph_stream_as", + ], "db/omnigraph.rs" => WriteProtocol::PhysicalOnly => ["cleanup"], "db/omnigraph.rs" => WriteProtocol::NativeRefControl => ["branch_create", "branch_create_as", "branch_create_from", "branch_create_from_as", "branch_delete", "branch_delete_as"], } @@ -387,6 +400,10 @@ const READ_ONLY_SURFACES: &[(&str, &str)] = &[ "db/omnigraph/stream_status.rs", "capture_served_graph_stream_status", ), + ( + "db/omnigraph/stream_correction.rs", + "show_graph_stream_data_block", + ), ( "db/omnigraph/stream_status.rs", "failpoint_stream_operational_status_for_test", @@ -427,10 +444,6 @@ const READ_ONLY_SURFACES: &[(&str, &str)] = &[ "db/omnigraph/stream_retirement.rs", "export_stream_dead_letter_payloads", ), - ( - "db/omnigraph/stream_correction.rs", - "show_stream_data_block", - ), ("exec/query.rs", "query"), ("exec/query.rs", "run_query_at"), ]; @@ -2541,6 +2554,114 @@ fn served_graph_stream_status_bridge_has_only_graph_logical_fields() { ); } +#[test] +fn served_graph_stream_controls_are_graph_only_and_redacted() { + let relative = "db/omnigraph/stream_management.rs"; + let path = engine_src_root().join(relative); + let contents = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + let ast = parse_rust_source(&contents, relative); + let expected_fields = BTreeMap::from([ + ( + "GraphStreamEnsureIndicesResult".to_string(), + BTreeSet::from(["changed".to_string(), "pending_index_count".to_string()]), + ), + ( + "GraphStreamOptimizeResult".to_string(), + BTreeSet::from([ + "changed".to_string(), + "pending_index_count".to_string(), + "requires_repair".to_string(), + ]), + ), + ( + "GraphStreamResumeResult".to_string(), + BTreeSet::from([ + "already_open_declarations".to_string(), + "enrolled_declarations".to_string(), + "profile_revision".to_string(), + "resumed_declarations".to_string(), + ]), + ), + ]); + let expected_methods = BTreeSet::from([ + "ensure_served_graph_stream_indices_as".to_string(), + "optimize_served_graph_stream_as".to_string(), + "resume_served_graph_stream_as".to_string(), + ]); + let mut seen_fields = BTreeMap::new(); + let mut seen_methods = BTreeSet::new(); + + for item in &ast.items { + match item { + Item::Struct(item) if item.ident.to_string().starts_with("GraphStream") => { + assert!(matches!(item.vis, Visibility::Public(_))); + assert!(has_doc_hidden(&item.attrs)); + let fields = item + .fields + .iter() + .map(|field| { + assert!( + matches!(field.vis, Visibility::Public(_)), + "transport must project the graph aggregate without private engine state" + ); + assert!( + ![ + "PendingIndex", + "TableIdentity", + "TableOptimizeStats", + "String", + ] + .iter() + .any(|forbidden| type_contains_identifier(&field.ty, forbidden)), + "graph stream result {} exposes selector or physical detail", + item.ident + ); + field.ident.as_ref().unwrap().to_string() + }) + .collect::>(); + seen_fields.insert(item.ident.to_string(), fields); + } + Item::Impl(item) if item.trait_.is_none() && is_omnigraph_type(&item.self_ty) => { + for member in &item.items { + let syn::ImplItem::Fn(function) = member else { + continue; + }; + if !matches!(function.vis, Visibility::Public(_)) { + continue; + } + assert!(function.sig.asyncness.is_some()); + assert!(has_doc_hidden(&function.attrs)); + assert_eq!( + function.sig.inputs.len(), + 2, + "served graph controls may accept only self plus graph actor authority" + ); + let Some(syn::FnArg::Typed(actor)) = function.sig.inputs.iter().nth(1) else { + panic!("served graph control must accept actor_id after self"); + }; + let syn::Pat::Ident(actor_name) = actor.pat.as_ref() else { + panic!("served graph control actor must be a simple binding"); + }; + assert_eq!(actor_name.ident, "actor_id"); + assert!(type_contains_identifier(&actor.ty, "str")); + seen_methods.insert(function.sig.ident.to_string()); + } + } + _ => {} + } + } + + assert_eq!(seen_fields, expected_fields); + assert_eq!(seen_methods, expected_methods); + for forbidden in ["table_key", "stable_table_id", "table_incarnation_id"] { + assert!( + !contents.contains(&format!("pub {forbidden}:")), + "served graph stream control must not expose '{forbidden}'" + ); + } +} + #[test] fn stream_driver_runtime_status_remains_advisory_and_failpoints_only() { let path = engine_src_root().join("db/omnigraph/stream_driver.rs"); diff --git a/crates/omnigraph/tests/memwal_stream.rs b/crates/omnigraph/tests/memwal_stream.rs index aa8cb737..6350b332 100644 --- a/crates/omnigraph/tests/memwal_stream.rs +++ b/crates/omnigraph/tests/memwal_stream.rs @@ -24,6 +24,7 @@ use arrow_array::{ }; use arrow_schema::{DataType, Field, Schema}; use async_trait::async_trait; +use base64::Engine; use fail::FailScenario; use futures::StreamExt; use futures::stream::BoxStream; @@ -3607,7 +3608,8 @@ struct F6aCandidateRuntimeFixture { } #[inline(never)] -async fn f6a_publish_mixed_candidate_cut() -> F6aCandidateRuntimeFixture { +async fn f6a_publish_mixed_candidate_cut(dead_letter_count: usize) -> F6aCandidateRuntimeFixture { + assert!(dead_letter_count > 0); let cluster = tempfile::tempdir().unwrap(); let graph = cluster.path().join("graphs/knowledge.omni"); let db = Arc::new( @@ -3662,36 +3664,44 @@ async fn f6a_publish_mixed_candidate_cut() -> F6aCandidateRuntimeFixture { assert!(next_witness.is_none()); let incarnation = incarnation.expect("lazy enrollment returns its durable incarnation"); - let body = [ - ndjson_score_line( + let mut body = Vec::new(); + for ordinal in 0..dead_letter_count { + let logical_id = if dead_letter_count == 1 { + "loser".to_string() + } else { + format!("loser-{ordinal:04}") + }; + body.extend(ndjson_score_line( &incarnation, - "loser", + &logical_id, 7, - "f6a10000-0000-4000-8000-000000000001", + &format!("{:08x}-0000-4000-8000-{:012x}", ordinal + 1, ordinal + 1), None, - ), - ndjson_score_line( - &incarnation, - "winner", - 9, - "f6a10000-0000-4000-8000-000000000002", - None, - ), - ] - .concat(); + )); + } + body.extend(ndjson_score_line( + &incarnation, + "winner", + 9, + "f6a10000-0000-4000-8000-000000000002", + None, + )); let outcomes = parse_ndjson_outcomes( served .failpoint_stream_ingest_ndjson_as_for_test(TABLE, vec![body], "agent:f6a-candidate") .await .expect("the mixed candidate batch crosses the durable NDJSON boundary"), ); - assert_eq!(outcomes.len(), 2); + assert_eq!(outcomes.len(), dead_letter_count + 1); for (ordinal, outcome) in outcomes.iter().enumerate() { assert_eq!(outcome["ordinal"], ordinal as u64); assert_eq!(outcome["status"], "durable"); assert!(outcome["stream_token"].as_str().is_some()); } - assert_ne!(outcomes[0]["stream_token"], outcomes[1]["stream_token"]); + assert_ne!( + outcomes[0]["stream_token"], + outcomes[dead_letter_count]["stream_token"] + ); let acknowledged = stream_fold_driver_status(&served); assert_process_local_driver_state(&acknowledged, "running"); @@ -3855,12 +3865,122 @@ async fn f6a_publish_correction_and_disable( fn candidate_runtime_composes_lazy_prepare_mixed_fold_terminal_correction_and_disable() { on_big_stack(|| async { let _scenario = FailScenario::setup(); - let fixture = f6a_publish_mixed_candidate_cut().await; + let fixture = f6a_publish_mixed_candidate_cut(1).await; let terminal_token = f6a_inspect_terminal_candidate(&fixture).await; f6a_publish_correction_and_disable(&fixture, &terminal_token).await; }); } +#[test] +#[serial] +fn dead_letter_logical_cursor_paginates_list_and_export_and_refuses_tampering() { + on_big_stack(|| async { + let _scenario = FailScenario::setup(); + let fixture = f6a_publish_mixed_candidate_cut(257).await; + let offline = reopen_enrolled(&fixture.dir).await; + let cluster_uri = fixture.dir.cluster_uri(); + + let first = + helpers::stream_authority::list_stream_dead_letters(&offline, &cluster_uri, None) + .await + .expect("the first bounded dead-letter page is readable"); + assert_eq!(first.entries.len(), 256); + assert_eq!(first.entries.first().unwrap().logical_id, "loser-0000"); + assert_eq!(first.entries.last().unwrap().logical_id, "loser-0255"); + let cursor = first + .next_cursor + .as_deref() + .expect("257 current terminal keys require a second page"); + + let decoded_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(cursor) + .expect("the bounded public cursor is canonical base64url"); + let mut decoded: serde_json::Value = serde_json::from_slice(&decoded_bytes).unwrap(); + let keys = decoded + .as_object() + .unwrap() + .keys() + .cloned() + .collect::>(); + assert_eq!( + keys, + BTreeSet::from([ + "authority_binding".to_string(), + "declaration_kind".to_string(), + "declaration_type".to_string(), + "format".to_string(), + "logical_id".to_string(), + ]) + ); + assert_eq!(decoded["declaration_kind"], "node"); + assert_eq!(decoded["declaration_type"], "Person"); + assert_eq!(decoded["logical_id"], "loser-0255"); + let rendered = String::from_utf8(decoded_bytes).unwrap(); + for forbidden in [ + "source_manifest_version", + "source_profile_revision", + "token_table_version", + "token_transaction_uuid", + "stable_table_id", + "table_incarnation_id", + ] { + assert!( + !rendered.contains(forbidden), + "public pagination cursor leaked {forbidden}: {rendered}" + ); + } + + decoded["logical_id"] = serde_json::json!("loser-0000"); + let tampered = base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(serde_json::to_vec(&decoded).unwrap()); + let error = helpers::stream_authority::list_stream_dead_letters( + &offline, + &cluster_uri, + Some(&tampered), + ) + .await + .expect_err("the hidden-cut binding must refuse a forged logical boundary"); + assert!( + matches!(error, OmniError::StreamingAuthorityMismatch { .. }), + "{error:?}" + ); + + let second = helpers::stream_authority::list_stream_dead_letters( + &offline, + &cluster_uri, + Some(cursor), + ) + .await + .expect("the authentic logical boundary selects the next page"); + assert_eq!(second.entries.len(), 1); + assert_eq!(second.entries[0].logical_id, "loser-0256"); + assert!(second.next_cursor.is_none()); + + let first_export = helpers::stream_authority::export_stream_dead_letter_payloads( + &offline, + &cluster_uri, + None, + ) + .await + .expect("payload export uses the same bounded logical cursor"); + assert_eq!(first_export.entries.len(), 256); + let export_cursor = first_export + .next_cursor + .as_deref() + .expect("payload export requires a second page"); + let second_export = helpers::stream_authority::export_stream_dead_letter_payloads( + &offline, + &cluster_uri, + Some(export_cursor), + ) + .await + .expect("payload export accepts its authentic next-page cursor"); + assert_eq!(second_export.entries.len(), 1); + assert_eq!(second_export.entries[0].authority.logical_id, "loser-0256"); + assert!(second_export.next_cursor.is_none()); + }); +} + /// Subprocess-only half of the F6b2 forced-termination acceptance cell. /// /// Keep this helper non-`serial`: the parent deliberately holds serial_test's @@ -4451,11 +4571,12 @@ async fn sealed_ensure_indices_refreshes_a_productive_lane_and_preserves_resume( assert_eq!(stream_lane(&db).await, sealed); assert_no_recovery_sidecars(&dir); - let pending = db - .failpoint_stream_sealed_ensure_indices_for_test("operator:sealed-indices") + let maintenance = db + .ensure_served_graph_stream_indices_as("operator:sealed-indices") .await .expect("checked SEALED maintenance must build the deferred id index"); - assert!(pending.is_empty()); + assert!(maintenance.changed); + assert_eq!(maintenance.pending_index_count, 0); let maintained = stream_lane(&db).await; let mut expected_maintained = sealed.clone(); expected_maintained.lifecycle_revision += 1; @@ -4478,11 +4599,12 @@ async fn sealed_ensure_indices_refreshes_a_productive_lane_and_preserves_resume( let before_no_work_status = db.stream_status().await.unwrap(); let before_no_work_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); - let pending = db - .failpoint_stream_sealed_ensure_indices_for_test("operator:sealed-indices") + let maintenance = db + .ensure_served_graph_stream_indices_as("operator:sealed-indices") .await .expect("a repeated checked maintenance call must recognize no work"); - assert!(pending.is_empty()); + assert!(!maintenance.changed); + assert_eq!(maintenance.pending_index_count, 0); let after_no_work_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); assert_eq!( db.stream_status().await.unwrap(), @@ -4830,18 +4952,13 @@ async fn sealed_indices_then_optimize_refresh_productive_authority_and_preserve_ ); assert_no_recovery_sidecars(&dir); - let stats = db - .failpoint_stream_sealed_optimize_for_test("operator:sealed-optimize") + let optimize = db + .optimize_served_graph_stream_as("operator:sealed-optimize") .await .expect("checked SEALED Optimize must compact the enrolled and ordinary siblings"); - for table_key in [TABLE, "node:Company"] { - assert!( - stats - .iter() - .any(|stat| stat.table_key == table_key && stat.committed), - "{table_key} must complete productive Optimize work: {stats:?}" - ); - } + assert!(optimize.changed); + assert_eq!(optimize.pending_index_count, 0); + assert!(!optimize.requires_repair); let maintained = stream_lane(&db).await; let mut expected_maintained = sealed.clone(); expected_maintained.lifecycle_revision += 1; @@ -4871,16 +4988,15 @@ async fn sealed_indices_then_optimize_refresh_productive_authority_and_preserve_ let before_no_work_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); let before_no_work_commit_count = db.list_commits(Some("main")).await.unwrap().len(); let repeated = db - .failpoint_stream_sealed_optimize_for_test("operator:sealed-optimize") + .optimize_served_graph_stream_as("operator:sealed-optimize") .await .expect("a repeated checked Optimize must recognize no data-table work"); assert!( - repeated - .iter() - .filter(|stat| stat.table_key == TABLE || stat.table_key == "node:Company") - .all(|stat| !stat.committed), - "the converged data tables must be true no-ops: {repeated:?}" + !repeated.changed, + "the converged graph must be a true no-op" ); + assert_eq!(repeated.pending_index_count, 0); + assert!(!repeated.requires_repair); let after_no_work_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); assert_eq!(db.stream_status().await.unwrap(), before_no_work_status); assert_eq!( @@ -5664,6 +5780,22 @@ async fn data_block_withdraw_uses_a_marker_only_base_effect_and_unstrands_the_dr .await; let cluster_uri = dir.cluster_uri(); let db = helpers::stream_authority::bind_checked_stream_runtime(db, &cluster_uri).await; + let resume_error = db + .resume_served_graph_stream_as("operator:blocked-graph-resume") + .await + .expect_err("graph resume must not bypass selected correction evidence"); + assert!( + resume_error + .to_string() + .contains("blocked by unresolved validation evidence"), + "{resume_error:?}" + ); + assert_eq!( + stream_lane(&db).await, + blocked, + "graph resume must leave the blocked drain untouched" + ); + assert_no_recovery_sidecars(&dir); let blocked_manifest_version = db .snapshot_of(ReadTarget::branch("main")) .await @@ -6365,6 +6497,150 @@ async fn sealed_resume_advances_epoch_replays_its_receipt_and_installs_the_write assert_no_recovery_sidecars(&dir); } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn graph_resume_converges_every_sealed_lane_without_exposing_a_selector() { + let _scenario = FailScenario::setup(); + let (dir, db) = init_f6b2_node_edge_fixture().await; + let cluster_uri = dir.cluster_uri(); + let db = helpers::stream_authority::bind_checked_stream_runtime(db, &cluster_uri).await; + + for (table_key, drain_id) in [ + (TABLE, "6a6a6a6a-6a6a-4a6a-8a6a-6a6a6a6a6a6a"), + (COMPANY_TABLE, "7a7a7a7a-7a7a-4a7a-8a7a-7a7a7a7a7a7a"), + ] { + let open = stream_lane_for(&db, table_key).await; + db.failpoint_stream_quiesce_for_test( + table_key, + drain_id, + open.lifecycle_revision, + "operator:graph-resume-fixture", + ) + .await + .expect("the selected fixture lane must reach SEALED"); + } + + let resumed = db + .resume_served_graph_stream_as("operator:graph-resume") + .await + .expect("one graph operation must resume every SEALED declaration"); + assert_eq!(resumed.enrolled_declarations, 3); + assert_eq!(resumed.resumed_declarations, 2); + assert_eq!(resumed.already_open_declarations, 1); + let status = db.stream_status().await.unwrap(); + assert!( + status.tables.iter().all(|lane| lane.lifecycle == "OPEN"), + "graph resume must leave no caller-selected SEALED lane: {:?}", + status.tables + ); + + let retry = db + .resume_served_graph_stream_as("operator:graph-resume") + .await + .expect("an exact graph retry must observe the converged authority"); + assert_eq!(retry.profile_revision, resumed.profile_revision); + assert_eq!(retry.enrolled_declarations, 3); + assert_eq!(retry.resumed_declarations, 0); + assert_eq!(retry.already_open_declarations, 3); + assert_no_recovery_sidecars(&dir); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn graph_resume_preflights_every_lane_before_resuming_a_sealed_lane() { + let _scenario = FailScenario::setup(); + let (dir, db) = init_f6b2_node_edge_fixture().await; + let cluster_uri = dir.cluster_uri(); + let db = helpers::stream_authority::bind_checked_stream_runtime(db, &cluster_uri).await; + + let person_open = stream_lane_for(&db, TABLE).await; + db.failpoint_stream_quiesce_for_test( + TABLE, + "8a8a8a8a-8a8a-4a8a-8a8a-8a8a8a8a8a8a", + person_open.lifecycle_revision, + "operator:graph-resume-preflight", + ) + .await + .expect("the first ordered lane must reach SEALED"); + let sealed_before = stream_lane_for(&db, TABLE).await; + assert_eq!(sealed_before.lifecycle, "SEALED"); + + let company_open = stream_lane_for(&db, COMPANY_TABLE).await; + db.failpoint_start_stream_open_after_fold_drain_for_test( + COMPANY_TABLE, + "9a9a9a9a-9a9a-4a9a-8a9a-9a9a9a9a9a9a", + company_open.lifecycle_revision, + "operator:graph-resume-preflight", + ) + .await + .expect("the later ordered lane must remain DRAINING"); + let draining_before = stream_lane_for(&db, COMPANY_TABLE).await; + assert_eq!(draining_before.lifecycle, "DRAINING"); + assert_no_recovery_sidecars(&dir); + + let error = db + .resume_served_graph_stream_as("operator:graph-resume-preflight") + .await + .expect_err("one DRAINING declaration must refuse the graph operation"); + assert!( + error + .to_string() + .contains("every drain to reach SEALED first"), + "{error:?}" + ); + assert_eq!( + stream_lane_for(&db, TABLE).await, + sealed_before, + "full preflight must precede the first per-lane resume effect" + ); + assert_eq!(stream_lane_for(&db, COMPANY_TABLE).await, draining_before); + assert_no_recovery_sidecars(&dir); +} + +#[tokio::test] +#[serial] +async fn graph_resume_authorizes_the_graph_even_when_no_lane_is_enrolled() { + struct GraphManageDecisionCounter { + calls: AtomicUsize, + } + + impl PolicyChecker for GraphManageDecisionCounter { + fn check( + &self, + action: omnigraph_policy::PolicyAction, + scope: &omnigraph_policy::ResourceScope, + actor: &str, + ) -> std::result::Result<(), omnigraph_policy::PolicyError> { + assert_eq!(action, omnigraph_policy::PolicyAction::StreamManage); + assert_eq!(scope, &omnigraph_policy::ResourceScope::Graph); + assert_eq!(actor, "operator:empty-graph-resume"); + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + let _scenario = FailScenario::setup(); + let (dir, db) = init_unenrolled_served_with_schema(TWO_TABLE_STREAM_SCHEMA).await; + let checker = Arc::new(GraphManageDecisionCounter { + calls: AtomicUsize::new(0), + }); + let policy: Arc = checker.clone(); + let db = match Arc::try_unwrap(db) { + Ok(db) => Arc::new(db.with_policy(policy)), + Err(_) => panic!("empty graph policy fixture must own the sole engine handle"), + }; + + let result = db + .resume_served_graph_stream_as("operator:empty-graph-resume") + .await + .expect("the graph-scoped no-op remains authorized and observable"); + assert_eq!(result.enrolled_declarations, 0); + assert_eq!(result.resumed_declarations, 0); + assert_eq!(result.already_open_declarations, 0); + assert_eq!(checker.calls.load(Ordering::SeqCst), 1); + assert_no_recovery_sidecars(&dir); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn frozen_driver_round_fences_resume_then_releases_its_empty_root_slot() { diff --git a/docs/dev/canon.md b/docs/dev/canon.md index f58160e9..f84242b6 100644 --- a/docs/dev/canon.md +++ b/docs/dev/canon.md @@ -572,10 +572,13 @@ state-lock-held, stopped/offline cluster adapter can show and correct one exact `REPLACE`/`WITHDRAW`. V19 adds the similarly stopped/offline `cluster stream dead-letter list|export` inspector over manifest-selected current authority. Explicit production enrollment, general lifecycle control, -`SEALED -> OPEN`, `AuthorityBlock` repair, direct SDK stream ingress/status, -and SDK/HTTP/OpenAPI lifecycle/maintenance parity remain inactive. F7b exposes +`AuthorityBlock` repair, direct SDK stream ingress/status/control, and public +per-declaration abort/rebind remain inactive. F7b exposes F6b6's checked read-only operational cut through one graph-redacted served -HTTP/OpenAPI route and remote CLI command. `ENABLED` uses runtime ownership, +HTTP/OpenAPI route and remote CLI command. F7c exposes selector-free graph-wide +`SEALED -> OPEN` resume and graph-wide checked `SEALED` EnsureIndices/Optimize +through HTTP/OpenAPI and the remote CLI; it exposes no declaration, table, or +lane selector. `ENABLED` uses runtime ownership, terminal `DISABLED | RETIRED` uses served-export ownership, and `DISABLING` uses explicit checked cluster- apply status authority. Immutable token/base parity, its bounded sample, @@ -1112,7 +1115,9 @@ on the proposer.) Direct SDK row admission/status and lifecycle mutation remain inactive; the Cedar vocabulary, enablement flip, and embedded manifest-only status are active, while F7b exposes F6b6's checked cut only as a graph-redacted served - status route and remote CLI command. + status route and remote CLI command. F7c exposes only selector-free + graph-wide resume and checked `SEALED` EnsureIndices/Optimize over those + served transports, reusing the existing recovery owners. The selected profile forbids OmniGraph from deleting raw `_mem_wal` paths and accepts monotonic storage plus loud provider exhaustion; a future managed-reclamation profile would require the missing operation to @@ -1206,7 +1211,8 @@ future bounded-storage claim, not blockers for this profile. B2a itself activated no schema or product surface. **Implemented B2 token/fold, bounded profile authority, graph-native served -row ingress, and hidden lifecycle core; public lifecycle controls remain inactive:** +row ingress, and hidden lifecycle core; bounded graph-wide controls active and +per-declaration lifecycle controls inactive:** Internal schema v9/config-v3/state-v2 adds canonical payload and token digests, trusted hidden row metadata, exact compare-and-chain/idempotency classification, same-generation token overlays, and the manifest-selected graph-global @@ -1233,10 +1239,12 @@ production quiescence owner. F7a exposes one graph-native mixed node/edge NDJSON route plus remote client/CLI/OpenAPI parity over the existing lazy private-lane prepare and resident driver; it adds no coordinator or format. Explicit public lane enrollment, general lifecycle control, `AuthorityBlock` -repair, direct SDK stream ingress/status, and SDK/HTTP/OpenAPI -lifecycle/maintenance parity remain inactive. F7b exposes F6b6's checked +repair, direct SDK stream ingress/status/control, and public per-declaration +abort/rebind remain inactive. F7b exposes F6b6's checked status core through a graph-redacted served HTTP/OpenAPI route and remote CLI -command. The core observes one read-only authority cut and retains honest +command. F7c exposes selector-free graph-wide resume plus graph-wide checked +`SEALED` EnsureIndices/Optimize through HTTP/OpenAPI and the remote CLI without +adding a coordinator, recovery grammar, or format strand. The core observes one read-only authority cut and retains honest unavailable states for cold replay, flushed LWW projection, exact canonical-main recovery-owned physical movement, and oldest uncovered age; all pending sidecars inside the accepted envelope remain visible and rebuild- @@ -1290,7 +1298,9 @@ its uncompacted-profile-cycle bounded NO-GO schedules no standalone production reconciler. F6b4 closes isolated dead-letter envelope evidence, F6b5 closes served export, F6b6 closes the internal checked -operational-status core, and F7b activates its graph-safe served transport. +operational-status core, F7b activates its graph-safe served transport, and F7c +activates graph-wide resume and checked `SEALED` maintenance without exposing a +lane control. Dead-letter inspection/export, future `AuthorityBlock` repair, and retirement remain cluster/offline-only, with no replay mutation or public @@ -1313,7 +1323,7 @@ resource budgets. | **R4: Manifest authority access grows with commit count.** Current-state resolution folds history; a selective index does not by itself bound the complete physical read. | Medium | `optimize` compacts internal tables (keeps periodically-optimized shipped paths flat where separately cost-gated). RFC-024 Gate A rejected durable heads because representative RustFS latest-manifest reads/bytes grow despite flat exact-BTREE row/range work. RFC-025 Gate 0 independently rejected checkpoint-registry activation: at local 10→1,000 on RC.1, uncompacted reconciled work and the eight-fragment tail stay flat, but compacted list/cleanup scan bytes grow 17,012→38,000 cold and 12,336→15,064 warm; exact-show bytes and operation counts also grow. Both RFCs are research-blocked; v8 retains the journal fold and internal-table *cleanup* remains deferred behind the resurrection watermark. | | **R5: Schema identity corruption or alias/identity drift.** Internal schema v5 introduced stable IDs/incarnation as durable authority; v6, v7, and v8 preserve them. | Medium | Open/init validate the SchemaIR domain and exact bidirectional IR↔manifest identity/path/alias contract; every active recovery envelope carries the identity pair; zero, duplicate, missing, or mismatched identity fails closed. | | **R6: Merge cost at divergence** — full-width classification and history-growing manifest folds. | Medium | Coherent coordinator scans plus retained probe handles reduced the pre-slice measured depth-5/depth-80 baseline from 59/651 manifest reads to 40/410 and cap the common fast-forward route at three internal opens and three scans, but the uncompacted-history slope remains. `merge_cost.rs` keeps both facts visible; O(delta) merge is blocked on a real deletion-delta source **(RFC-027)**; fragment adoption is **(draft RFC-0001)**. | -| **R7: The public graph firehose has a deliberately narrow operating envelope** — F7a serves absent/`OPEN` lanes in the single-writer checked runtime and F7b serves a graph-redacted checked status cut, while lifecycle/maintenance transport and overlapping-process ownership remain incomplete. | Medium | MemWAL is the strategic substrate. The bounded worker provides watcher-plus-post-fence acknowledgement and conservative replay; its legal near-cap shape closes under logical dense-slice accounting. Schema v9–v18 establish compare-and-chain tokens, checked profile/lifecycle authority, maintenance/rebind, retirement, and exact DataBlock correction. Current v19/recovery-v21 adds bounded terminal folds, current `DEAD_LETTERED` authority, ordinary successors, cluster/offline inspection/export, and three-disposition retirement. F7a activates graph-native served row ingress by reusing that core and the resident driver; it adds no new coordinator or format. F6b5 separately activates exact-terminal served export; F6b6 implements checked operational status internally; F7b exposes only its logical graph projection over HTTP/OpenAPI and the remote CLI; F6b7 adds paired failpoints-only selected token-index evidence and records the uncompacted-profile-cycle bounded NO-GO. Exact evidence remains in [RFC-026](../rfcs/0026-memwal-streaming-ingest.md) and the [testing map](testing.md). Public resume/rebind and maintenance transport, direct SDK parity, and the remaining guardrail acceptance stay open. Managed reclamation and a whole-root history budget are optional later work; a public exact enrollment receipt plus reversible admission seal gates broader overlapping-process topology. | +| **R7: The public graph firehose has a deliberately narrow operating envelope** — F7a serves absent/`OPEN` lanes in the single-writer checked runtime, F7b serves a graph-redacted checked status cut, and F7c serves graph-wide resume and checked `SEALED` maintenance; per-declaration lifecycle/rebind, direct SDK control, and overlapping-process ownership remain incomplete. | Medium | MemWAL is the strategic substrate. The bounded worker provides watcher-plus-post-fence acknowledgement and conservative replay; its legal near-cap shape closes under logical dense-slice accounting. Schema v9–v18 establish compare-and-chain tokens, checked profile/lifecycle authority, maintenance/rebind, retirement, and exact DataBlock correction. Current v19/recovery-v21 adds bounded terminal folds, current `DEAD_LETTERED` authority, ordinary successors, cluster/offline inspection/export, and three-disposition retirement. F7a activates graph-native served row ingress by reusing that core and the resident driver; it adds no new coordinator or format. F6b5 separately activates exact-terminal served export; F6b6 implements checked operational status internally; F7b exposes only its logical graph projection over HTTP/OpenAPI and the remote CLI; F7c composes the frozen recovery-v15/v16/v17 owners behind selector-free graph controls; F6b7 adds paired failpoints-only selected token-index evidence and records the uncompacted-profile-cycle bounded NO-GO. Exact evidence remains in [RFC-026](../rfcs/0026-memwal-streaming-ingest.md) and the [testing map](testing.md). Public per-declaration resume/abort/rebind, direct SDK parity, and the remaining guardrail acceptance stay open. Managed reclamation and a whole-root history budget are optional later work; a public exact enrollment receipt plus reversible admission seal gates broader overlapping-process topology. | | **R8: Some operations lack enforced memory/time budgets.** | Medium | Known gap, narrowed and accepted for RFC-023. Its direct-substrate instrument rejected the first whole-delta fenced adopt (~447 MB peak at 100K × 256 versus ~74 MB Append), and the first corrected production 10K series failed at 30.0× / 108,625,920 bytes overhead; both negative results remain evidence. Mutation/Load now refuses a keyed table above 8,192 rows / 32 MiB before arm, while BranchMerge uses a recovery-enrolled chain with the same per-chunk bounds and a 1,024-transaction ceiling. The inductive certificate route removes the general diff, temporary delta, target preflight, and target join without weakening that chain. Final five-pair production medians passed at 31/8 ms (3.875×) for 10K and 136/35 ms (~3.886×) for 100K; maximum signed paired RSS overheads were 24,297,472 and 32,604,160 bytes. Inclusive row/transaction ceilings, byte refusal (including materialized blobs), operation-wide validation retention, exact source/target incarnation revalidation, second-generation certificate composition, and both between-chunk recovery directions are pinned; other operations still need explicit bounds. | | **R9: Local-FS conditional-write emulation** (`write_text_if_match` check-then-act gap). | Low | All current callers sit behind the cluster lock protocol; S3 uses true conditional puts; close before admitting any lock-free caller. | | **R10: Doc/spec drift as the system grows** — this document included. | Low | Maintenance contract (same-PR doc updates, `check-agents-md.sh` link CI, "don't lie" stale markers); this canon defers to area docs by construction. | @@ -1373,9 +1383,10 @@ Live design questions, each owned by an RFC or a known gap — not a wishlist: export, F6b6's internal checked operational status, and F6b7's failpoints-only paired token-index decision instrument are implemented. Its uncompacted- profile-cycle bounded NO-GO schedules no standalone production reconciler; - F7b's graph-redacted served status transport is implemented; direct-SDK - status, lifecycle/maintenance transport, and the F6 guardrail remainder stay - later. + F7b's graph-redacted served status transport and F7c's selector-free + graph-wide resume plus checked `SEALED` EnsureIndices/Optimize transports are + implemented; direct-SDK status/control, per-declaration lifecycle/rebind, + and the F6 guardrail remainder stay later. No active producer creates `AuthorityBlock`, so its repair stays fail-closed. RFC-024's heads, RFC-025's retention, and later RFC-026 phases remain @@ -1410,7 +1421,7 @@ The plan of record is the RFC-022…028 family (all under | [0023 — Key-conflict fencing](../rfcs/0023-key-conflict-fencing.md) | Substrate-native keyed-write fencing via Lance's unenforced-PK filter; fleet/format activation barrier | **Implemented** (2026-07-15) | | [0024 — Durable table heads](../rfcs/0024-durable-table-heads.md) | Materialized head-row research; the first exact-BTREE candidate bounded scan work but failed the full latest-manifest/object-byte cost gate | **Research blocked** | | [0025 — Checkpoint-pinned retention](../rfcs/0025-checkpoint-retention.md) | Named checkpoints as authoritative retention roots, materialized as Lance tags; current in-manifest registry lookup rejected by Gate 0 | **Research-blocked** | -| [0026 — MemWAL streaming ingest](../rfcs/0026-memwal-streaming-ingest.md) | Durability-first streaming writes: bounded watcher-plus-post-fence acknowledgement, compare-and-chain token authority, unbounded retain-all, checked profile/lifecycle control, recovery-v15–v20 maintenance/rebind/retirement/correction, and v19/recovery-v21 bounded terminal dead-letter folds plus three-disposition retirement. Hidden lifecycle/rebind/dead-letter, narrow cluster/offline inspection/correction/rebuild exits, F6b5 exact-terminal served export, F6b6 checked operational status, F7b's graph-redacted HTTP/OpenAPI/remote-CLI projection, and the F6b7 failpoints-only token-index decision instrument (uncompacted-profile-cycle bounded NO-GO; no standalone production reconciler) are implemented; F7a graph-native served row ingress is active, while public lane enrollment/lifecycle and maintenance transports remain inactive | **Draft; hidden protocol through F5b, F6 evidence, F7a graph row activation, and F7b graph status transport implemented; remaining lifecycle/maintenance activation stays staged** | +| [0026 — MemWAL streaming ingest](../rfcs/0026-memwal-streaming-ingest.md) | Durability-first streaming writes: bounded watcher-plus-post-fence acknowledgement, compare-and-chain token authority, unbounded retain-all, checked profile/lifecycle control, recovery-v15–v20 maintenance/rebind/retirement/correction, and v19/recovery-v21 bounded terminal dead-letter folds plus three-disposition retirement. Hidden lifecycle/rebind/dead-letter, narrow cluster/offline inspection/correction/rebuild exits, F6b5 exact-terminal served export, F6b6 checked operational status, F7b's graph-redacted HTTP/OpenAPI/remote-CLI projection, F7c's selector-free graph-wide resume and checked `SEALED` EnsureIndices/Optimize transports, and the F6b7 failpoints-only token-index decision instrument (uncompacted-profile-cycle bounded NO-GO; no standalone production reconciler) are implemented; F7a graph-native served row ingress is active, while public lane enrollment, per-declaration/general lifecycle/abort/rebind, and direct SDK control remain inactive | **Draft; hidden protocol through F5b, F6 evidence, and F7a–F7c graph activation implemented; remaining per-declaration lifecycle/direct-SDK activation stays staged** | | [0027 — Lineage merge deltas](../rfcs/0027-lineage-merge-deltas.md) | O(delta) merge classification from row-version lineage | Research-blocked | Deliberately split, not one mega-format: identity, key fencing, head rows, diff --git a/docs/dev/invariants.md b/docs/dev/invariants.md index caf218ff..e743d4e7 100644 --- a/docs/dev/invariants.md +++ b/docs/dev/invariants.md @@ -251,7 +251,7 @@ shared side and remain concurrent with one another. | Constructive mutations | In-memory `MutationStaging`, one end-of-query table commit per touched table, then one manifest publish | [writes.md](writes.md), [execution.md](execution.md) | | Keyed writes | Every current-format node/edge table declares exact non-null physical `id` as Lance's unenforced PK from creation, using the v6-introduced and later-preserved exact-`id` fence. General production strict insert and upsert use the sealed exact-`id`, forced-v2 MergeInsert adapter; strict insert exact-probes its pinned parent before minting `omnigraph.insert_absence=v1`, while an all-new upsert may mint it only when completed effect statistics prove one attempt inserted every source row with zero updates, deletes, or skipped duplicates. Certificate admission is optional: BranchMerge uses the shortcut only when every transaction in the complete contiguous source interval carries v1 and its persisted operation is the full pure-insert `Update` shape (exact parent, no removed or updated fragments, nonempty new fragments with `physical_rows`, no field or generation rewrites, `RewriteRows`, exact-`id` filter, full nested schema preorder, and matching physical-row total). It then rechecks both source and target native ref incarnations and passes owned batches through an opaque capability whose one production mint site is structurally guarded. The proven publisher stages immutable fragments with `InsertBuilder`, replaces the uncommitted Append operation with that filter-bearing `Update`, and performs zero target preflights, target merge joins, or committed Appends. Missing, cleaned, unknown, or malformed proof uses the general ordered diff. Mutation/Load remains one keyed transaction per table and rejects more than 8,192 rows or 32 MiB before arm; BranchMerge keeps its bounded v4 chain and exact-recovery limits. Raw Lance graph writers are unsupported, and the certificate is an internal non-cryptographic capability rather than an authenticity mechanism. The final production cost gate passed at 10K (3.875× median; 24,297,472-byte max paired RSS overhead) and 100K (~3.886×; 32,604,160 bytes) | [RFC-023](../rfcs/0023-key-conflict-fencing.md), [writes.md](writes.md), [execution.md](execution.md) | | Deletes | Staged like inserts/updates (`stage_delete` via Lance 7.0 `DeleteBuilder::execute_uncommitted`, MR-A) — no inline HEAD advance; mixed insert/update/delete in one query rejected by D2 as a deliberate boundary (constructive XOR destructive per query; compose via separate mutations or a branch) | [query-language.md](../user/queries/index.md), [writes.md](writes.md) | -| Streaming ingest | RFC-026 adopts Lance MemWAL and remains Draft; F7a graph-native served row ingress is active while public management remains inactive. V7–v9 established enrollment, the bounded worker, compare-and-chain token authority, and exact base/token fold; v10–v18 added checked profile/lifecycle authority plus recovery-owned profile, claim, drain, resume, maintenance, rebind, retirement, and DataBlock correction. Current v19/token-schema-v3/recovery-v21 adds deterministic mixed/all-diverted folds, one bounded dead-letter object, current `DEAD_LETTERED` sequencing, ordinary-successor correction, selected-token stopped/offline inspection/export, and three-disposition retirement. F4/F5 provide the hidden caller path, resident folding, deterministic disable, and terminal diversion. F6a–F6b4 provide composition, process/lifecycle, token-cost, and production dead-letter-envelope evidence. F6b1's checked exact-version move-only export cut blocks current terminal authority and owns one nonwaiting root slot; F6b5 now connects it to the existing served HTTP/remote-CLI/OpenAPI export route with pre-header validation, incremental exact-version scans using approximate Lance targets, strict 64-KiB chunks, a complete queue-envelope/process queue reservation under a deadline, backpressure, and disconnect-safe release. The queue reservation is not a whole-response or RSS bound. F6b6 adds an engine-internal checked read-only operational cut over physical lane, token, recovery, advisory-driver, and rebuild evidence. Expensive immutable token/base parity, the bounded sample, index coverage, and selected lifecycle-ledger proofs run once without writer gates; a short second phase validates the exact selected manifest/recovery cut and repeats only mutable witnesses under root/profile/lane fences. It binds the exact runtime owner for `ENABLED`, served-export owner for terminal `DISABLED | RETIRED`, or explicit checked cluster-apply status owner for `DISABLING`. Within the hard status envelope, every pending recovery sidecar is reported and rebuild-blocking; exceeding any discovery bound refuses the whole status; only exact canonical-main recovery ownership makes physical movement explicit unavailable evidence, while overlap or unexplained movement is `StreamStatusChanged`. Cold replay, flushed LWW projection accounting, and exact oldest-uncovered age are unavailable rather than inferred. The public nonblocking embedded `stream_status` remains manifest-only. F7b exposes the checked cut only through a doc-hidden graph-redacted engine bridge, `GET /graphs/{graph_id}/stream/status`, remote `stream status`, and OpenAPI; the wire omits physical and recovery identities. Embedded/direct export of an enrolled graph still refuses; the served graph must be exact terminal `DISABLED | RETIRED`, and `RETIRED` export carries verified provenance. Checked offline disable remains the supported production quiescence owner; same-schema rebind requires terminal `DISABLED` and exact `SEALED` authority, while enrolled schema evolution uses checked export into a fresh graph. F7a exposes one graph-only mixed node/edge NDJSON route and remote command behind a strong derived graph-authority ETag, reusing lazy private-lane preparation and the resident driver without a new coordinator or format. Public lane enrollment, general lifecycle/rebind, fresh read, and generation GC remain inactive. F6b7 adds a failpoints-only paired selected-index decision instrument for current-token and receipt-key work. Its configured-RustFS result is a bounded NO-GO only for the uncompacted profile-cycle fixture, so no standalone production token-index reconciler is scheduled; the remaining guardrail matrix stays open. No active path produces reserved `AuthorityBlock`, so its repair stays fail-closed. | [RFC-026](../rfcs/0026-memwal-streaming-ingest.md), [writes.md](writes.md) | +| Streaming ingest | RFC-026 remains Draft, but the graph-native production path is active: mixed node/edge NDJSON ingest, checked graph-redacted status, selector-free graph-wide resume, and graph-wide SEALED EnsureIndices/Optimize are served through HTTP/OpenAPI and the remote CLI. They reuse Lance MemWAL plus OmniGraph’s existing manifest coordinator and frozen recovery-v14/v15/v16/v17 grammars; this surface adds no WAL, persisted queue, coordinator, or format strand. Resume preflights the complete enrolled cut, refuses drains/strict blocks before planned effects, skips `OPEN`, and deterministically composes per-declaration recovery-v15 for the `SEALED` remainder. Maintenance composes the existing one-publication recovery-v16/v17 owners and returns aggregate-only results. Physical datasets, lanes, bindings, claims, Lance versions, and recovery identities remain private. Checked offline cluster apply still owns `DISABLING` and quiescence; stopped/offline graph-level block correction, dead-letter inspection/export, and irreversible terminal-authority retirement remain the narrow cluster controls. Served export still requires an exact terminal `DISABLED | RETIRED` cut and carries retirement provenance when applicable. Public per-declaration enrollment/lifecycle/abort, rebind, fresh reads, and generation GC remain inactive. F6b7’s uncompacted-profile-cycle result still schedules no standalone token-index reconciler. | [RFC-026](../rfcs/0026-memwal-streaming-ingest.md), [writes.md](writes.md) | | Branch create/delete | `__manifest` `BranchContents` is the single logical authority. Lance create is physically two-phase, so OmniGraph prevalidates names, enforces path-prefix-disjoint live graph names, reclaims an absent-ref clone-only tree, and uses a bounded completion classifier; delete removes authority before tree cleanup, so an absent ref is success and derived tree reclaim may converge later. Neither control emits graph lineage. Under schema/source-target/all-table gates, each control uses one operation-local post-gate manifest/namespace capture rather than refreshing the handle-local coordinator around table-gate acquisition; successful ref movement explicitly invalidates derived read caches. Per-table forks are derived state, reclaimed best-effort with `cleanup` as backstop. A target-scoped unresolved sidecar may be made unreachable by deletion and is then audit-discarded by recovery; graph-global SchemaApply still blocks. Public create/create-from/delete also acquire the stream-export root slot nonwaitingly, preventing native named-branch delete/recreate ABA while an immutable cut is live | [branches-commits.md](../user/branching/index.md), [maintenance.md](../user/operations/maintenance.md), [writes.md](writes.md) | | Cleanup retention | Explicit cleanup derives exact `keep` cutoffs from Lance's available version list, caps each main-table GC cutoff at the oldest exact main version inherited by any live lazy graph branch, and refuses uncovered main HEAD drift. Lance protects native per-table branch refs itself. The graph-wide live-reference preflight fails closed before the first table GC, after which individual table failures remain fault-isolated | [maintenance.md](../user/operations/maintenance.md), [writes.md](writes.md) | | Optimize visibility | One operation-local accepted catalog and fresh main snapshot are planned under schema → main → all-table gates. Every productive table shares one identity-bearing v9 recovery envelope; bounded-parallel physical effects become visible through at most one monotonic manifest/lineage CAS. Complete crash residuals roll forward together and partial residuals compensate before visibility. Optimize's payload remains a bounded maintenance adapter rather than an exact caller-minted Lance transaction proof, so it is supported within the single-writer-process recovery boundary. Exact provenance is deferred until Lance exposes a stable public caller-controlled maintenance transaction API and OmniGraph has distributed recovery fencing | [maintenance.md](../user/operations/maintenance.md), [writes.md](writes.md), [RFC-022](../rfcs/0022-unified-write-path.md) | @@ -493,15 +493,19 @@ them explicit. ordinary load there. Never load the artifact over the enrolled source, and never describe physical rebind as changing accepted schema. Public/production rebind remains inactive. - Public enrollment and general resume/abort and lifecycle control remain - required before a public profile. The + Public enrollment, per-declaration resume/abort, and general lifecycle + control remain inactive. The only production resume is graph-wide: it + preflights the complete enrolled cut and composes the frozen recovery-v15 + lane operation without exposing a declaration/table selector. The `stream_ingest` / `stream_manage` Cedar vocabulary is already registered, `stream_manage` gates the enablement writer, and the embedded SDK already exposes durable manifest-only status; a `GraphHistoryBudget` or retained-storage watermark belongs only to a future - bounded/managed profile, not this one. F7a's graph-native served row path and - F7b's graph-redacted checked status path are active; public lane enrollment - and lifecycle/maintenance CLI/HTTP/OpenAPI surfaces remain inactive. The stopped/offline retirement + bounded/managed profile, not this one. F7a's graph-native served row path, + F7b's graph-redacted checked status path, and the graph-wide checked + resume/SEALED-maintenance CLI/HTTP/OpenAPI controls are active. Those + controls reuse recovery-v15/v16/v17 unchanged and expose no lane/table + address. The stopped/offline retirement, exact `DataBlock` show/correct, and current dead-letter list/export commands are active emergency controls. Hidden F4 caller-shaped ingress/prepare, format-neutral F5a automatic diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 0e338bdf..5dc3d3c5 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -7,7 +7,7 @@ This file is the always-on map of the test surface. **Consult it before every ta | Crate | Path | Style | |---|---|---| | `omnigraph` (engine) | `crates/omnigraph/tests/` | Integration tests (one file per behavior area — see the table below), fixture-driven, share `tests/helpers/mod.rs` | -| `omnigraph-cli` | `crates/omnigraph-cli/tests/` | Per-area suites (post-modularization): `cli_cluster.rs` (cluster command surface + operator-actor cascade, including strict stream-block/dead-letter grammar, scope, plan parsing, and effect-free offline preflight), `cli_cluster_e2e.rs` (spawned-binary lifecycle compositions — lost-state re-import recovery, out-of-band drift, graph-root destruction, multi-graph mixed-disposition convergence), `cli_data.rs` (load/read/change/branch/commit/export/snapshot/policy/embed/maintenance + operator format cascade), `cli_schema_config.rs` (init/config, schema plan/apply), `cli_queries.rs`, `parity_matrix.rs` (RFC-009 Phase 1: the embedded-vs-remote referee — every forked verb run against both arms with matched Cedar policy and the same actor, scrubbed-JSON + exit-code equality; divergences are pinned in its `KNOWN_DIVERGENCES` ledger, never silently repaired), `system_local.rs` (full-cycle cluster lifecycle with a spawned `--cluster` server, applied-policy enforcement over HTTP, keyed-credential auth, operator aliases), `system_remote.rs`, `crossversion_upgrade.rs` (genuine historical source→CURRENT rebuild/refusal cells plus the required adjacent harness — see below); share `tests/support/mod.rs` (hermetic `OMNIGRAPH_HOME` by default) | +| `omnigraph-cli` | `crates/omnigraph-cli/tests/` | Per-area suites (post-modularization): `cli_cluster.rs` (cluster command surface + operator-actor cascade, including strict stream-block/dead-letter grammar, scope, plan parsing, and effect-free offline preflight), `cli_cluster_e2e.rs` (spawned-binary lifecycle compositions — lost-state re-import recovery, out-of-band drift, graph-root destruction, multi-graph mixed-disposition convergence), `cli_data.rs` (load/read/change/branch/commit/export/snapshot/policy/embed/maintenance + operator format cascade), `cli_schema_config.rs` (init/config, schema plan/apply), `cli_queries.rs`, `parity_matrix.rs` (RFC-009 Phase 1: the embedded-vs-remote referee — every forked verb run against both arms with matched Cedar policy and the same actor, scrubbed-JSON + exit-code equality; divergences are pinned in its `KNOWN_DIVERGENCES` ledger, never silently repaired), `system_local.rs` (full-cycle cluster lifecycle with a spawned `--cluster` server, applied-policy enforcement over HTTP, keyed-credential auth, operator aliases, and the real-binary graph-first firehose golden journey), `system_remote.rs`, `crossversion_upgrade.rs` (genuine historical source→CURRENT rebuild/refusal cells plus the required adjacent harness — see below); share `tests/support/mod.rs` (hermetic `OMNIGRAPH_HOME` by default) | | `omnigraph-control-authority` | in-source `#[cfg(test)] mod tests` | Concrete-storage, lock-derived checked authority: offline confirmation/actor/operation binding, state-CAS and graph/declaration/profile-revision validation, normalized graph-root binding, non-cloneable runtime guards, and one process-local writer registration per cluster graph. F6b1 pins a distinct served-export guard for exact terminal `DISABLED | RETIRED` state and proves it shares that registration without becoming writer authority | | `omnigraph-cluster` | mostly in-source `#[cfg(test)] mod tests`; `tests/failpoints.rs` (feature-gated); `tests/s3_cluster.rs` (bucket-gated full lifecycle on object storage) | Cluster config parser, local JSON state diff, state CAS/lock handling/recovery, read-only validate/plan/status plus explicit refresh/import graph observations, config-only apply (content-addressed payload publish, disposition gating, composite-digest convergence, idempotent re-apply), catalog payload verification (status read-only, refresh drift + self-heal), failpoint crash-mid-apply / CAS-race coverage, graph create/schema/delete lifecycle, policy binding and serving snapshots, v11 streaming-profile ownership, authority-retirement preflight, and stopped/offline stream-control preflight. The current dead-letter owner pins actor/offline/applied-streaming/declaration/state-lock binding before selected-token list or payload export; it is inspection-only and exposes no served route. F6b1 adds only the exact-terminal served-export binding consumed at boot | | `omnigraph-server` | `crates/omnigraph-server/tests/` | Per-area suites (post-modularization): `auth_policy.rs`, `data_routes.rs`, `schema_routes.rs`, `stored_queries.rs`, `multi_graph.rs` (cluster-mode boot — converged serving, policy binding wiring, boot refusals — + the concurrent branch-ops matrix), `boot_settings.rs` (mode inference, PolicySource), `s3.rs` (bucket-gated: single-graph serving + config-free `--cluster s3://` boot), `openapi.rs` (OpenAPI drift / regeneration); share `tests/support/mod.rs`. F5a changes no route: `serve` starts checked-runtime resident fold supervisors only after listener bind and joins every selected graph concurrently after Axum graceful shutdown; engine/failpoint owners pin the scheduler behavior. F6b1 boot consumes the terminal served-export guard and installs the hidden engine authority; it adds no handler, route, or OpenAPI surface | @@ -648,8 +648,9 @@ apply failpoint seams are also registered as named read-only surfaces; in `stream_status.rs` and guards the named surfaces' direct durable-call shape. It does not prove the full composed call graph contains no publication primitive. The operational-status cells' unchanged manifest version and absent -sidecar/effect assertions own that behavioral claim. Public CLI/HTTP/OpenAPI/ -SDK transport is not covered because it does not exist yet. +sidecar/effect assertions own that behavioral claim. The served F7b +route/CLI/OpenAPI projection is covered by the server and CLI owners below; a +direct-SDK checked-runtime transport remains intentionally absent. The v11 bounded profile-authority slice extends different existing owners. `omnigraph-control-authority` in-source tests pin lock-derived offline @@ -683,16 +684,29 @@ test in the v11 profile-authority slice activated ingress, enrollment, claim, or ordinary lifecycle mutation. The current v19 hidden lifecycle owners are `memwal_stream.rs` plus the v14/v15/v16/v17/v18/v19/v20/v21 in-source recovery/lifecycle suites described above. F7a extends the existing graph-row -owner and adds server/CLI/OpenAPI coverage; those tests still do not activate -an ordinary lifecycle transport surface. +owner and adds server/CLI/OpenAPI coverage. F7c adds only graph-wide controls: +`memwal_stream.rs` owns all-OPEN no-op, mixed OPEN/SEALED convergence, +multi-declaration resume, and DRAINING/strict-block preflight-before-effect; +the existing F3b/F3c cells remain the recovery-v16/v17 maintenance authority. +`forbidden_apis.rs` classifies the three doc-hidden graph bridges by their +frozen recovery owners. Server auth/multi-graph/OpenAPI tests own bodyless +routes, graph-scoped `stream_manage`, default deny, aggregate DTOs, and error +redaction. CLI client/plane/output tests own selector-free grammar, exact POST +paths, bearer propagation, aggregate rendering, and status next-action hints. +`system_local.rs::local_cluster_firehose_golden_journey_uses_graph_only_controls` +closes the public composition through the real CLI and server binaries: +cluster-owned enablement, four independently visible mixed node/edge folds, +checked status, stopped/offline disable and re-enable, productive sealed +EnsureIndices and Optimize, convergent graph-wide resume, and a visible +successor ingest. It never names a table, dataset, or lane. The historical B1/B2a and private B2-common slices added no parser/server/ingest-CLI tests because they had no public row surface. F7a now extends the existing server/OpenAPI, CLI, Cedar, shutdown, no-raw-GC, and -provider-failure ownership for graph ingress without activating lane controls. -Future public management must additionally cover persistent revisioned -status/quiesce/resume/abort-drain, bounded terminal management receipts, -attribution, and strict correction. The selected retain-all profile has no byte/object/file/history +provider-failure ownership for graph ingress. F7c activates graph-wide resume +and sealed maintenance without exposing a lane control; per-declaration +enrollment/resume/abort-drain, public rebind, and direct-SDK checked control +remain future work. The selected retain-all profile has no byte/object/file/history quota; its tests must instead prove that provider exhaustion is loud and cannot drop an acknowledgement or bypass recovery/manifest visibility. Storage watermarks and graph-history admission controls belong only to a future B2b diff --git a/docs/dev/writing-path-state-of-affairs.md b/docs/dev/writing-path-state-of-affairs.md index 81149752..d5353211 100644 --- a/docs/dev/writing-path-state-of-affairs.md +++ b/docs/dev/writing-path-state-of-affairs.md @@ -117,8 +117,10 @@ admission, drains a finite manifest-selected lane cut, and resumes that stored plan after restart. F7a activates one graph-native served-ingress route and remote CLI/OpenAPI contract; declaration routing and lazy lane enrollment stay private. F7b activates one graph-redacted checked status route and remote CLI. -Public lane enrollment, general lifecycle resume/abort/rebind, maintenance -transport, and direct-SDK status remain inactive; +F7c activates selector-free graph-wide `SEALED → OPEN` resume and graph-wide +checked `SEALED` EnsureIndices/Optimize through HTTP/OpenAPI and the remote CLI. +Public lane enrollment, per-declaration/general lifecycle control including +abort-drain, rebind, and direct-SDK status/control remain inactive; retirement, `stream block show|correct`, and `stream dead-letter list|export` are the narrow cluster-only CLI exceptions. @@ -128,9 +130,11 @@ graph bridge at `POST /graphs/{graph_id}/stream/ingest`; its engine mechanics remain private and feature-gated tests own their fault matrix. The graph-scoped Cedar vocabulary, public manifest-only status, checked profile-control boundary, and engine-internal checked operational-status cut exist. F7b exposes -the cut's graph-redacted served projection; there is still no `@stream` syntax, -public lane-enrollment/fold API, enrolled-lane drain/resume/correction surface, -direct-SDK checked status, or fresh-read mode. +the cut's graph-redacted served projection. F7c exposes only graph-wide resume +and checked `SEALED` maintenance, without a declaration/table/lane selector; +there is still no `@stream` syntax, public lane-enrollment/fold API, +per-declaration drain/resume/abort or rebind surface, direct-SDK checked +status/control, or fresh-read mode. This page answers four practical questions: @@ -314,9 +318,9 @@ These operations do not create a side door around the protocol: | Recovery | Same-process handles share ordered queues for the same canonical local root or identical normalized opaque remote/custom URI. Stream-admission domains are acquired outside profile-shared → schema → branch → stream-token → table gates where applicable. Mutation/load, SchemaApply, BranchMerge, EnsureIndices, Optimize, active v14–v21 stream operations, StreamProfileChange, and `refresh` heal roll-forward-only; Repair and Cleanup refuse pending recovery, while branch controls use specialized barriers. Read-write open performs the quiesced full sweep, including v14 enrollment/claim/fold/terminal-receipt, v15 resume/abort, v16 SEALED EnsureIndices, v17 SEALED Optimize, v18 SEALED physical rebind, historical v19 two-disposition retirement, v20 exact `DataBlock` correction, v21 terminal fold/three-disposition retirement, and v13 profile-receipt completion; read-only open never repairs and refuses unresolved stream recovery. Historical recovery-v10 enrollment, v11 B1 fold, v12 lifecycle-v2 fold, and v14 resume/maintenance/retirement/rebind are refused. A resolved graph-lineage intent is audited internally with the original actor when present; retirement instead uses its selected immutable receipt/profile chain and emits no graph lineage or `RecoveryAudit`. No-effect cleanup need not create graph lineage. | | RFC-026 Phase A (v7 foundation) | Internal schema v7 introduced identity-keyed lifecycle rows and exact empty main/unsharded enrollment. At that Phase-A-only boundary, any lifecycle row—including `SEALED`—rejected base-table, schema, maintenance, repair-adoption, and recovery effects under a process-local admission lease because no drain/fold witness-update adapter existed. Native branch create/delete alone could proceed at `SEALED` because it did not move table HEAD; `OPEN`/`DRAINING` refused it. Enrollment and open-time validation rejected named-branch overlap and any uncovered lifecycle/MemWAL mismatch. Current v19 preserves these foundation guarantees while adding the guarded lifecycle-v3 claim/fold/quiesce/resume transitions and explicit SEALED EnsureIndices/Optimize/rebind bridges. | | RFC-026 Phase B1 (historical private core) | Internal schema v8/config-v2 added one root-scoped, cross-handle serialized worker and a hard-bounded 8,192-row/32-MiB logical dense-slice Arrow no-roll generation. Watcher success proves durability; a clean `DurableBatchAck` additionally requires the same `ShardWriter::check_fenced()` to succeed immediately afterward. Fence loss, epoch-read failure, owner-task failure, or deadline ambiguity is post-invocation `AckUnknown` plus worker retirement. Reopen/replay is conservative and exact drain proof precedes quiesced abort. Gate R0's deterministic legal high-entropy near-cap cell exposed sparse scanner arrays; logical-slice charging plus dense copies repaired it. Physical RSS is evidence only. V9 preserves those worker/closure mechanics, but recovery-v11 itself is historical only. | -| RFC-026 B2 token/fold core + profile/lifecycle authority + F7a graph ingress | Internal schema v9/config-v3/state-v2 adds canonical payload/token digests, hidden trusted row metadata, exact idempotency and compare-and-chain classification, same-generation token overlays, and post-admission authority recapture. `_stream_tokens.lance` is graph-global durable sequencing state selected only by its manifest witness. V11/profile-v2 adds checked stopped/offline and runtime ownership, resumable `DISABLING`, fail-closed `RETIRED`, and exact recovery-v13 profile receipts. V12/lifecycle-v3 replaces inline histories with fixed-size ledger heads; recovery-v14 covers enrollment, writer claims, ordinary/drain folds, and terminal management receipts. V13–v20 add private resume, SEALED maintenance/rebind, authority retirement, and DataBlock correction. Current v19/token-schema-v3/recovery-v21 adds deterministic terminal diversion and extends retirement to `WITHDRAWN | DEAD_LETTERED`. The private one-lane core can cycle `OPEN → DRAINING → SEALED → OPEN`; checked offline apply is the supported `OPEN/DRAINING → SEALED` owner. F7a exposes graph-native served row admission over absent or `OPEN` lanes while keeping lane enrollment private. F7b exposes a graph-redacted read-only projection of F6b6's checked cut over HTTP/OpenAPI and the remote CLI. Public lane enrollment, resume/rebind control, maintenance transport, and direct-SDK status remain inactive. F6b5 separately activates exact-terminal served HTTP/remote-CLI/OpenAPI export with bounded transport ownership. F6b6 implements the underlying checked read-only operational cut with explicit checked `DISABLING` authority and complete pending-sidecar inventory inside a hard advisory envelope: 256 matching direct `.json` sidecars, 256 irrelevant direct-or-nested objects encountered below the prefix, 4 MiB of cumulative input-anchored URI bytes across all encountered objects, 32 MiB per sidecar body, and 32 MiB of cumulative bodies. Exceeding any bound is a typed refusal, never partial status. The cut retains honest recovery/unavailable physical evidence and unavailable cold-replay/flushed/oldest-age fields. Retirement, DataBlock inspection/correction, and selected current dead-letter list/export remain narrow cluster-only CLI controls. F6 owns remaining measurements and guardrail acceptance. | +| RFC-026 B2 token/fold core + profile/lifecycle authority + F7 graph surface | Internal schema v9/config-v3/state-v2 adds canonical payload/token digests, hidden trusted row metadata, exact idempotency and compare-and-chain classification, same-generation token overlays, and post-admission authority recapture. `_stream_tokens.lance` is graph-global durable sequencing state selected only by its manifest witness. V11/profile-v2 adds checked stopped/offline and runtime ownership, resumable `DISABLING`, fail-closed `RETIRED`, and exact recovery-v13 profile receipts. V12/lifecycle-v3 replaces inline histories with fixed-size ledger heads; recovery-v14 covers enrollment, writer claims, ordinary/drain folds, and terminal management receipts. V13–v20 add private resume, SEALED maintenance/rebind, authority retirement, and DataBlock correction. Current v19/token-schema-v3/recovery-v21 adds deterministic terminal diversion and extends retirement to `WITHDRAWN | DEAD_LETTERED`. The private one-lane core can cycle `OPEN → DRAINING → SEALED → OPEN`; checked offline apply is the supported `OPEN/DRAINING → SEALED` owner. F7a exposes graph-native served row admission over absent or `OPEN` lanes while keeping lane enrollment private. F7b exposes a graph-redacted read-only projection of F6b6's checked cut over HTTP/OpenAPI and the remote CLI. F7c exposes selector-free graph-wide resume plus graph-wide checked `SEALED` EnsureIndices/Optimize over HTTP/OpenAPI and the remote CLI, reusing recovery-v15/v16/v17 without a new coordinator or format. Public lane enrollment, per-declaration/general lifecycle control including abort-drain, rebind, and direct-SDK status/control remain inactive. F6b5 separately activates exact-terminal served HTTP/remote-CLI/OpenAPI export with bounded transport ownership. F6b6 implements the underlying checked read-only operational cut with explicit checked `DISABLING` authority and complete pending-sidecar inventory inside a hard advisory envelope: 256 matching direct `.json` sidecars, 256 irrelevant direct-or-nested objects encountered below the prefix, 4 MiB of cumulative input-anchored URI bytes across all encountered objects, 32 MiB per sidecar body, and 32 MiB of cumulative bodies. Exceeding any bound is a typed refusal, never partial status. The cut retains honest recovery/unavailable physical evidence and unavailable cold-replay/flushed/oldest-age fields. Retirement, DataBlock inspection/correction, and selected current dead-letter list/export remain narrow cluster-only CLI controls. F6 owns remaining measurements and guardrail acceptance. | | Lance access | One process-wide `ObjectStoreRegistry` reuses clients. Each `Omnigraph` handle owns its cached data-table `Session`; one process-wide zero-cache control `Session` opens mutable tips. Only the object-store registry is shared between the data and control sessions. This is “cache the past, never the present,” not one global cached session. | -| Maintenance | EnsureIndices stages exact missing-index transactions. Its private checked-runtime recovery-v16 path can refresh an enrolled table only at exact `SEALED`; ambient calls remain refused. Optimize coordinates graph-wide compaction/index work under one bounded recovery envelope; its private recovery-v17 path applies the same checked-runtime, exact-`SEALED` pointer/proof refresh while ambient calls remain refused for enrolled tables. Recovery-v18 separately rebinds an exact `SEALED` lane without opening it. Periodic optimize compacts `__manifest`, but unmaintained history-dependent paths are not globally flat. | +| Maintenance | EnsureIndices stages exact missing-index transactions. Its checked-runtime recovery-v16 path can refresh an enrolled table only at exact `SEALED`; Optimize coordinates graph-wide compaction/index work under one bounded recovery envelope, and recovery-v17 applies the same checked-runtime, exact-`SEALED` pointer/proof refresh. F7c exposes both only as selector-free graph-wide HTTP/OpenAPI/remote-CLI controls with aggregate results; ambient/direct calls remain refused for enrolled tables. Recovery-v18 separately rebinds an exact `SEALED` lane without opening it and remains private. Periodic optimize compacts `__manifest`, but unmaintained history-dependent paths are not globally flat. | Selected enforced limits are 8,192 rows / 32 MiB per keyed Mutation/Load table, 8,192 rows / 32 MiB of logical dense-slice Arrow bytes for the complete private @@ -339,7 +343,7 @@ retry rules, recovery classification, and the full owned limits. | [023 — Key-conflict fencing](../rfcs/0023-key-conflict-fencing.md) | **Implemented** | Internal schema v6 introduced exact-`id` PK metadata, closed keyed routing, typed conflicts, bounded replay, rebuild/refusal, and accepted performance evidence; v19 preserves that contract. | | [024 — Durable table heads](../rfcs/0024-durable-table-heads.md) | **Research-blocked** | The first in-manifest BTREE candidate has a specified logical contract and flat indexed row/range work, but fails the complete physical-I/O gate. No head rows or heads format are active. | | [025 — Checkpoint retention](../rfcs/0025-checkpoint-retention.md) | **Research-blocked** | Lance tag/pin semantics pass, but the proposed in-manifest registry access shape is not history-flat after compaction. No checkpoint rows, `ogcp_` production tags, API, or cleanup integration are active. | -| [026 — MemWAL streaming ingest](../rfcs/0026-memwal-streaming-ingest.md) | **Draft; graph-native served row ingress and graph-redacted status, hidden lane/fold/lifecycle core, narrow cluster/offline controls, and exact-terminal served export implemented** | Schema v9 introduced config-v3/state-v2 token authority plus recovery-v12 exact base+token fold. V11–v21 add checked profile/lifecycle ownership, recovery-covered claims/quiesce/resume/maintenance/rebind/correction/retirement, deterministic terminal folds, exact retry/ordinary successor, and selected-token inspection. Historical recovery payloads retain their exact grammar. F6b3/F6b7 provide uncovered and failpoints-only paired reconciled token-index evidence; F6b7 records a bounded standalone-reconciler NO-GO for the uncompacted profile-cycle fixture, so recovery-owned maintenance remains open only to new evidence (including graph-manifest-compacted/checked-Optimize coupling). F6b4 closes the dead-letter envelope; F6b5 connects the move-only exact-terminal cut to existing HTTP/remote-CLI/OpenAPI export with strict chunk/queue ownership; F6b6 implements the checked read-only operational core; F7b exposes its graph-redacted HTTP/OpenAPI/remote-CLI projection. Direct-SDK status and remaining guardrails stay open. Checked offline disable is the supported production quiescence owner. F7a activates one graph-level mixed node/edge NDJSON row route and remote CLI/OpenAPI contract over the existing lazy private-lane prepare and resident driver. Public lane enrollment, general resume/abort/rebind, served lifecycle/maintenance surfaces, and direct-SDK status remain inactive. B2b managed reclamation remains optional future work. | +| [026 — MemWAL streaming ingest](../rfcs/0026-memwal-streaming-ingest.md) | **Draft; graph-native served row ingress, graph-redacted status, graph-wide resume/SEALED maintenance, hidden lane/fold/lifecycle core, narrow cluster/offline controls, and exact-terminal served export implemented** | Schema v9 introduced config-v3/state-v2 token authority plus recovery-v12 exact base+token fold. V11–v21 add checked profile/lifecycle ownership, recovery-covered claims/quiesce/resume/maintenance/rebind/correction/retirement, deterministic terminal folds, exact retry/ordinary successor, and selected-token inspection. Historical recovery payloads retain their exact grammar. F6b3/F6b7 provide uncovered and failpoints-only paired reconciled token-index evidence; F6b7 records a bounded standalone-reconciler NO-GO for the uncompacted profile-cycle fixture, so recovery-owned token-index maintenance remains open only to new evidence (including graph-manifest-compacted/checked-Optimize coupling). F6b4 closes the dead-letter envelope; F6b5 connects the move-only exact-terminal cut to existing HTTP/remote-CLI/OpenAPI export with strict chunk/queue ownership; F6b6 implements the checked read-only operational core; F7b exposes its graph-redacted HTTP/OpenAPI/remote-CLI projection. F7c exposes selector-free graph-wide resume and checked `SEALED` EnsureIndices/Optimize over the same served transports. Direct-SDK status/control and remaining guardrails stay open. Checked offline disable is the supported production quiescence owner. F7a activates one graph-level mixed node/edge NDJSON row route and remote CLI/OpenAPI contract over the existing lazy private-lane prepare and resident driver. Public lane enrollment, per-declaration/general lifecycle control including abort-drain, and rebind remain inactive. B2b managed reclamation remains optional future work. | | [027 — Lineage merge deltas](../rfcs/0027-lineage-merge-deltas.md) | **Research-blocked** | The desired O(delta) classifier and fallback contract are specified. Selective live-row and deletion-delta discovery are not yet bounded, so `OrderedTableCursor` remains the correctness path. | | [028 — Stable schema identity](../rfcs/0028-stable-schema-identity.md) | **Implemented** | Rename-stable IDs, table incarnation, identity-derived paths, schema/recovery integration, and strict rebuild activation were introduced in v5 and remain active in v19. | @@ -571,8 +575,10 @@ durable fold attribution. The historical final-v8-source → CURRENT cell proves strict two-way refusal and rebuild fidelity while excluding hidden trusted metadata from export. The RFC remains Draft. F7a activates graph-native served rows and F7b activates a graph-redacted checked driver/rebuild-status projection; -lifecycle, maintenance, direct-SDK status, and raw physical-status transports -remain inactive. F6b5 is the other narrow transport exception: the existing +F7c activates selector-free graph-wide resume and checked `SEALED` +EnsureIndices/Optimize. Per-declaration/general lifecycle control, rebind, +direct-SDK status/control, and raw physical-status transports remain inactive. +F6b5 is the other narrow transport exception: the existing served HTTP/remote-CLI/OpenAPI export route accepts an exact cluster-served terminal cut with bounded transport ownership. F6b6 implements the checked operational-status core behind an engine-internal diff --git a/docs/releases/v0.10.0.md b/docs/releases/v0.10.0.md index 1ab72a87..e8990d86 100644 --- a/docs/releases/v0.10.0.md +++ b/docs/releases/v0.10.0.md @@ -4,8 +4,8 @@ Cluster-served graphs with `streaming: true` now accept mixed node/edge NDJSON at `POST /graphs/{graph_id}/stream/ingest` while each selected declaration lane -is absent (and prepared lazily) or `OPEN`. F7a does not expose resume, so a -`SEALED` lane after disable/re-enable remains closed. The URL, Cedar resource, remote +is absent (and prepared lazily) or `OPEN`. A `SEALED` declaration remains +closed to ingest until the explicit graph-wide resume control opens it. The URL, Cedar resource, remote client, and `omnigraph stream ingest --data ` command are graph-level; Lance datasets, table incarnations, and MemWAL lanes remain private runtime details. The engine reuses the existing lazy enrollment, recovery, bounded @@ -69,30 +69,44 @@ checked offline disable owner invokes that core after publishing `DISABLING`, visits one finite manifest-derived lane cut in deterministic order, and adopts an existing `OPEN_AFTER_FOLD` drain without minting a second occurrence. A selected `DataBlock` leaves the apply durably pending until stopped/offline -correction and an apply retry. There is still no standalone public lifecycle -verb. The crate-private evidence path can also recovery-cover an explicit -`SEALED → OPEN` resume or a guarded `DRAINING → OPEN` abort; that is not a -supported operator surface. Existing `SEALED` enrollments remain fenced after +correction and an apply retry. General per-declaration lifecycle and guarded +`DRAINING → OPEN` abort controls remain private. Recovery-v15's explicit +`SEALED → OPEN` operation is exposed only through the selector-free graph-wide +resume described below. Existing `SEALED` enrollments remain fenced after disable, so only the no-lane case restores direct content writes. An enrolled source must satisfy ordinary export's clean preflight or use the exact same-format retirement/export authority-discard exit; neither logical rebuild path transfers live sequencing authority. Disabling itself never discards that authority. -The private F3b evidence seam can run content-preserving EnsureIndices on +After re-enabling and restarting a graph with sealed declarations, +`POST /graphs/{graph_id}/stream/resume` and `omnigraph stream resume` reopen the +whole graph. There is no declaration/table selector. The control preflights the +complete enrolled cut, refuses a drain or strict block before effects, skips +already-open declarations, and converges across retry by processing the sealed +remainder in deterministic order. It reuses recovery-v15 per declaration and +adds no coordinator, recovery grammar, or storage-format strand. + +The F3b path can run content-preserving EnsureIndices on canonical main through the exact checked serving runtime and a `stream_manage` actor, but only when every enrolled productive table is exactly `SEALED`. Recovery-v16 publishes index pointers with refreshed lifecycle HEAD witnesses, proof digests, and revisions atomically. This naturally idempotent maintenance call has no caller operation ID or token-ledger ManagementReceipt. Ambient -EnsureIndices remains refused for enrolled tables. +EnsureIndices remains refused for enrolled tables. It is exposed only as the +graph-wide `POST /graphs/{graph_id}/stream/maintenance/ensure-indices` and +`omnigraph stream maintenance ensure-indices` controls; responses are +aggregate-only. -The private F3c seam adds the corresponding checked `SEALED` Optimize path. +The F3c path adds the corresponding checked `SEALED` Optimize path. Recovery-v17 owns Optimize's internally committing compaction/index-maintenance effects, records their achieved physical HEADs, and publishes the table pointers with refreshed lifecycle proofs in one manifest CAS. A true no-work Optimize stays effect-free. Ambient Optimize remains refused for enrolled -tables; all CLI/HTTP/OpenAPI maintenance surfaces remain unavailable. +tables. The graph-wide +`POST /graphs/{graph_id}/stream/maintenance/optimize` and +`omnigraph stream maintenance optimize` controls expose it without a +type/table selector or physical fragment details. The private F3d seam adds recovery-covered physical rebind for an exact `SEALED` lane. It retains the old binding and claim history, creates a fresh @@ -122,7 +136,8 @@ could create `WITHDRAWN` remained inactive. The F3f slice adds the stopped/offline cluster-only `stream block show|correct` surface for one exact `DataBlock` while the lane remains `DRAINING`. Show reconstructs bounded, cursor-stable correction evidence from the retained WAL -cut. Correct requires an actor, block token, lifecycle revision, correction +cut. The opaque block token resolves the internal declaration; neither command +accepts a type/table/lane selector. Correct requires an actor, block token, lifecycle revision, correction UUID, and an ordered REPLACE/WITHDRAW plan. Recovery-v20 owns one pre-minted base transaction and one combined token-successor plus immutable correction/ management-receipt transaction. It may recreate only the latter after proving @@ -134,6 +149,13 @@ PRESENT attribution, releases that exact block, and leaves the drain in `DRAINING`. This is correction, not resume: the ordinary drain owner must still finish the lane to `SEALED`. +Block and dead-letter JSON now cross a cluster-owned projection boundary. They +retain accepted-schema node/edge declarations, logical correction/terminal +evidence, and opaque cursors, but omit table keys, stable/incarnation +identifiers, stream/fold/recovery coordinates, Lance token-table versions, and +object-store descriptors. Public diagnostics use stable graph-level messages +rather than rendering raw engine errors. + The F5b slice advances the current development format to internal schema v19, upgrades `_stream_tokens.lance` to schema v3, and adds recovery-v21. One deterministic bounded fold publishes valid winners and records every losing diff --git a/docs/rfcs/0026-memwal-streaming-ingest.md b/docs/rfcs/0026-memwal-streaming-ingest.md index 84b8662d..c3b3531a 100644 --- a/docs/rfcs/0026-memwal-streaming-ingest.md +++ b/docs/rfcs/0026-memwal-streaming-ingest.md @@ -81,11 +81,16 @@ commit identifiers. An unmanaged graph or changed/busy cut fails closed; the route never invents or truncates evidence. One nonwaiting observation slot per graph root and one per serving process bound concurrent immutable scans. It changes no format, recovery grammar, coordinator, or lifecycle authority. -Public lane enrollment, general lifecycle/rebind verbs, `AuthorityBlock` repair, +F7c activates selector-free graph-wide `SEALED → OPEN` resume and graph-wide +checked `SEALED` EnsureIndices/Optimize through HTTP/OpenAPI and the remote CLI. +It composes the frozen recovery-v15/v16/v17 owners, exposes aggregate graph +results rather than declaration/table/lane identities, and changes no format, +recovery grammar, or coordinator. Public lane enrollment, per-declaration/ +general lifecycle verbs including abort-drain, public rebind, `AuthorityBlock` repair, the standalone production token-index reconciler deferred by F6b7's uncompacted- profile-cycle bounded NO-GO, the F6b-remainder -guardrail acceptance, and SDK/HTTP/OpenAPI lifecycle or maintenance surfaces -remain inactive. Direct SDK operational-status parity also remains inactive. +guardrail acceptance, and direct SDK operational-status/control parity remain +inactive. Stream-aware served export remains the separate terminal-cut transport. Retirement, DataBlock show/correct, and current dead-letter list/export are exposed only by narrow stopped/offline cluster controls. @@ -289,7 +294,8 @@ fairness, and maintenance/rebind/resume cells, and F6b3 closes the uncovered- tail current-token hit/miss and terminal-page instrument. F6b7 later adds the paired token-index decision instrument; at that boundary operational-status transport and the remaining guardrails kept F7 forbidden. F7a/F7b later -activated graph row ingress and graph-redacted checked status; +activated graph row ingress and graph-redacted checked status; F7c later +activated selector-free graph-wide resume and checked `SEALED` maintenance; F6b4 separately closes the isolated dead-letter envelope evidence and F6b5 closes bounded served export. **F6b1 checked immutable export-cut slice implemented:** 2026-08-02 — an exact @@ -514,6 +520,16 @@ graph, or returns a partial inventory. One nonwaiting observation slot per graph root and one per serving process bounds concurrent immutable scans. The embedded manifest-only status API is unchanged. This slice changes no format, recovery grammar, coordinator, or Lance operation. +**F7c graph-wide lifecycle/maintenance controls implemented:** 2026-08-04 — +bodyless `POST /graphs/{graph_id}/stream/resume` and remote `stream resume` +preflight the complete enrolled cut, refuse `DRAINING` or strict blocks before +effects, skip `OPEN`, and deterministically compose recovery-v15 over the +`SEALED` remainder. The graph-wide EnsureIndices and Optimize routes and remote +commands compose the existing checked `SEALED` recovery-v16/v17 owners and +return aggregate-only results. All three controls require `stream_manage` and +expose no declaration, table, lane, binding, or physical-result selector. This +slice adds no coordinator, recovery grammar, or format strand; per-declaration +resume/abort, public rebind, and direct SDK control remain inactive. **Author track:** Maintainer design series **Depends on:** [RFC-022](0022-unified-write-path.md)'s unified write and generic recovery-sidecar protocol, plus @@ -598,9 +614,10 @@ acknowledgements, or a format stamp. The Phase A implementation authorized by that result has now activated internal schema v7 and the bounded production foundation described in §12.2: recoverable empty enrollment, durable lifecycle authority, process-local admission/exclusion, and strict format -refusal/rebuild. It still exposes no production enrollment entry point and -cannot append or acknowledge a row. RFC-026 therefore remains draft and public -streaming remains inactive until the later gates close. +refusal/rebuild. At that boundary it exposed no production enrollment entry +point and could not append or acknowledge a row. RFC-026 remains draft, but +later F7a–F7c now expose graph-native served ingress, graph-redacted status, +graph-wide resume, and checked `SEALED` maintenance. The implementation remains deliberately split. Historical **Phase B1** supplied one admission-bounded, no-roll generation from admission through crash replay @@ -631,20 +648,21 @@ remains reachable only through feature-gated, doc-hidden engine test seams and is not a product surface. V11's profile protocol v2 and exact recovery-v13 `StreamProfileChange` remain unchanged. -The remaining public enrollment/quiesce, `AuthorityBlock` correction, physical -status, rebind, and product-parity contracts in §4.1–§4.4 and §4.6 still -apply. Narrow stopped/offline `DataBlock` correction, selected-current-token +The remaining public per-declaration enrollment/quiesce/abort, +`AuthorityBlock` correction, physical status, rebind, and direct-SDK parity +contracts in §4.1–§4.4 and §4.6 still apply. Narrow stopped/offline +`DataBlock` correction, selected-current-token dead-letter list/export, and terminal authority retirement are the cluster-only operator exceptions. `GraphHistoryBudget`, physical-storage admission, and aggregate receipt-capacity reservations do not. The graph-scoped Cedar vocabulary and embedded -manifest-only status are now active under §4.7, but no schema intent, -production first use, served SDK/HTTP/OpenAPI write surface, general lifecycle -control, or public same-key `AckUnknown` retry contract is implemented. `DISABLING` -persists an exact restart/resume plan and drain-only continuation. The hidden -core can drain a non-`SEALED` lane, but no production cluster/CLI/HTTP/SDK -adapter invokes that path yet, so public disable still waits for -already-`SEALED` lanes. +manifest-only status are active under §4.7. F7a adds served graph ingress, F7b +graph-redacted checked status, and F7c selector-free graph-wide resume plus +checked `SEALED` EnsureIndices/Optimize; schema intent, per-declaration/general +lifecycle control, direct-SDK control, and a public same-key `AckUnknown` retry +contract remain unimplemented. `DISABLING` persists an exact restart/resume +plan and drain-only continuation, and checked offline cluster apply is the +production owner that drains its finite cut to `SEALED`. ### 0.2 Gate R0: historical bounded-retention result and current disposition @@ -1029,17 +1047,20 @@ receipt-bearing export exit; v18/recovery-v20 adds the separate stopped/offline exact `DataBlock` show/correct exit; and current v19/token-schema-v3/ recovery-v21 adds deterministic terminal diversion plus three-disposition retirement. Those format slices themselves expose no production row caller or -maintenance transport; F7a later activates the graph row caller without a +maintenance transport; F7a later activates the graph row caller and F7c the +selector-free graph-wide resume/checked `SEALED` maintenance controls without a format change. The graph-scoped `stream_ingest` / `stream_manage` Cedar vocabulary and embedded manifest-only read-only status are active under §4.7; v11 profile mutation additionally requires checked cluster-control/runtime ownership. -Supported explicit enrollment/quiesce/rebind, general lifecycle control, -`AuthorityBlock` repair, direct SDK status, and maintenance transport parity -remain future gates. F7a activates graph-native served row admission, while F7b -exposes a graph-redacted HTTP/OpenAPI/remote-CLI projection of F6b6's checked -read-only operational-status core. +Supported explicit enrollment/quiesce/rebind, per-declaration/general lifecycle +control including abort-drain, `AuthorityBlock` repair, and direct SDK +status/control remain future gates. F7a activates graph-native served row +admission, F7b exposes a graph-redacted HTTP/OpenAPI/remote-CLI projection of +F6b6's checked read-only operational-status core, and F7c exposes graph-wide +resume plus checked `SEALED` EnsureIndices/Optimize over the same served +transports. **B2a unbounded retain-all** is the selected first profile: it deletes no MemWAL object and performs no physical-storage admission or accounting. **B2b** is the deferred managed-reclamation profile through the @@ -3087,9 +3108,10 @@ blocking; exceeding any discovery bound refuses the whole status. Only an exact canonical-main recovery participant outcome makes physical movement an unavailable projection rather than a change. F7b exposes only its graph- redacted served HTTP/OpenAPI and remote-CLI projection; direct-SDK checked - status remains inactive. The private P2 prepare proof has no row -body or product surface; P3–P7 remain unimplemented, and enabling the profile -activates no ingest surface. Ambient embedded SDK and direct-store + status remains inactive. The private P2 prepare proof has no enrollment +product surface. F7a activates graph ingress and F7c activates only graph-wide +resume and checked `SEALED` maintenance; per-declaration lifecycle/rebind and +direct SDK control remain inactive. Ambient embedded SDK and direct-store callers cannot mint the checked authority required to mutate it. **Profile boundaries.** Main-only; unsharded; one resident writer root-wide; one @@ -3946,8 +3968,9 @@ queue; a stalled receiver backpressures production; and completion, disconnect, and error release every reservation. The queue/root limits and preflight/stall/disconnect handler cells co-land with HTTP/remote-CLI/OpenAPI export parity. F7a later activates public -graph row ingress and F7b graph-redacted checked status; F7 still owns -lifecycle, maintenance, direct SDK parity, and the rest of their served parity. +graph row ingress, F7b graph-redacted checked status, and F7c selector-free +graph-wide resume plus checked `SEALED` EnsureIndices/Optimize. Per-declaration +lifecycle/abort/rebind and direct SDK control remain staged. The resulting artifact may initialize a fresh target through normal cluster control, never load over the enrolled source. @@ -3988,6 +4011,10 @@ over the existing candidate runtime. It deliberately leaves lifecycle, maintenance, and checked-status transport to later F7 slices. Export remains the earlier narrow HTTP/remote exception. +F7b later activates checked status, and F7c activates only graph-wide resume +and checked `SEALED` EnsureIndices/Optimize. Per-declaration lifecycle/abort, +rebind, and direct SDK control remain inactive. + The F7a activation PR extends F2's already-public cluster-ownership, direct-mutation-refusal, and v10→v11 rebuild baseline with the activated stream operating contract. CLI reference, server, policy, and error docs add the @@ -4649,11 +4676,12 @@ ordinary commits carry no fold summary. Provenance is never inferred from MemTable positions or WAL cursor statistics. This row/fold design is active in v9; v18 adds the separate stopped/offline exact `DataBlock` correction exit, and current v19 adds terminal dead-letter evidence plus versioned attribution. -Public row exposure, general lifecycle control, `AuthorityBlock` repair, -direct SDK checked-status parity, and the remaining F6 acceptance evidence -remain later gates. Graph-redacted operational status is active through the -served HTTP/OpenAPI/CLI surface; the checked physical status core remains -internal. +F7a activates public graph row exposure, F7b graph-redacted operational status, +and F7c selector-free graph-wide resume plus checked `SEALED` +EnsureIndices/Optimize through HTTP/OpenAPI and the remote CLI. General +per-declaration lifecycle/abort/rebind, `AuthorityBlock` repair, direct SDK +checked-status/control parity, and the remaining F6 acceptance evidence remain +later gates; the checked physical status core remains internal. The registered Cedar vocabulary and embedded manifest-only status do not widen the private row/fold seam. @@ -5658,10 +5686,13 @@ The dense copy releases those buffers and restores the intended 32-MiB logical dense-slice Arrow closure check; physical RSS remains a separate evidence tripwire. The near-cap cell now closes. This amendment changed no product surface. The subsequent v9 slice implements §4.1's private token/attribution -and §4.4's base+token fold core. Explicit enrollment, lifecycle management, -correction, public row admission, and transport parity remain inactive; the -Cedar vocabulary and embedded manifest-only status are active, and F6b6 adds -the separate internal checked operational-status core. §4.5.1's B2a profile is +and §4.4's base+token fold core. At that boundary explicit enrollment, +lifecycle management, correction, public row admission, and transport parity +were inactive. Later F7a–F7c activate graph row admission, graph-redacted +status, graph-wide resume, and checked `SEALED` maintenance; per-declaration +lifecycle/abort/rebind and direct SDK control remain inactive. The Cedar +vocabulary and embedded manifest-only status are active, and F6b6 adds the +separate internal checked operational-status core. §4.5.1's B2a profile is implemented, while §4.5.2's B2b managed-reclamation profile remains optional and inactive. @@ -5763,9 +5794,11 @@ This B2a result itself added no schema or product surface. The subsequent private B2-common row/fold slice activated schema v9; F3f later added the narrow stopped/offline DataBlock exit, F7a later activated the graph-native served row caller plus HTTP/remote-CLI/OpenAPI parity, and F7b activated the graph-redacted -checked status route and remote CLI. General lifecycle control, -`AuthorityBlock` repair, direct SDK status, and maintenance transport remain -§12.6 work. F6b6 implements the checked operational core; the +checked status route and remote CLI. F7c activated selector-free graph-wide +resume and checked `SEALED` EnsureIndices/Optimize through those served +transports. Per-declaration/general lifecycle control including abort-drain, +rebind, `AuthorityBlock` repair, and direct SDK status/control remain §12.6 +work. F6b6 implements the checked operational core; the authorization/manifest-status slice shipped earlier under §4.7. ### 12.6 Private B2-common implementation and remaining public/B2b gates @@ -5783,8 +5816,11 @@ and three-disposition retirement through recovery-v21. F7a activates the selected profile's graph-native served row bridge, lazy private prepare, cancellation ownership, and HTTP/remote-CLI/OpenAPI parity. F7b activates the graph-redacted checked operational-status HTTP/OpenAPI route and remote CLI. -Explicit lane enrollment, general lifecycle controls, `AuthorityBlock` repair, -direct SDK stream ingress/status, and maintenance transport remain inactive. +F7c activates selector-free graph-wide resume and checked `SEALED` +EnsureIndices/Optimize through HTTP/OpenAPI and the remote CLI. Explicit lane +enrollment, per-declaration/general lifecycle controls including abort-drain, +public rebind, `AuthorityBlock` repair, and direct SDK stream ingress/status/ +control remain inactive. F6b6 implements the underlying checked operational-status core. Cold-replay and flushed-LWW accounting plus exact oldest-uncovered age are explicitly unavailable. `DISABLING` uses explicit checked cluster- @@ -5800,11 +5836,12 @@ The design does not waive the persistent escape requirement: a user must never be left with a table that ordinary writers refuse but cannot be corrected, quiesced, or rebuilt. -- **Graph row surface activated in F7a; controls remain inactive:** +- **Graph row surface activated in F7a; graph-wide controls activated in F7c:** `@stream(mode="upsert", on_reject="strict")` production first use, the served client, HTTP, remote CLI, and OpenAPI all route through the same private core. `stream_ingest` has one production graph caller; - `stream_manage` still reaches only existing cluster/offline controls. Existing + `stream_manage` also gates F7c's selector-free graph-wide resume and checked + `SEALED` EnsureIndices/Optimize. Existing `/ingest` behavior must remain compatible. The full surface requires embedded/remote command parity; the selected §4.7 profile instead tests served/remote success against the exact @@ -5862,7 +5899,10 @@ ordinary writers refuse but cannot be corrected, quiesced, or rebuilt. historical and refused under v9. The graph-global token gate and release-all-gates/restart rule cover every manifest writer. - **Still inactive at the product boundary:** embedded/direct SDK row ingress, - public lifecycle/status/maintenance transports, and public lane enrollment. + checked status/control parity, public per-declaration lifecycle/abort/rebind, + and public lane enrollment. Graph-redacted status and selector-free graph-wide + resume/checked `SEALED` maintenance are active only through HTTP/OpenAPI and + the remote CLI. Any later SDK or control surface must preserve the graph-only boundary and the token/redaction rules above; it cannot promote the private lane adapter into a user-visible API. @@ -5875,10 +5915,12 @@ ordinary writers refuse but cannot be corrected, quiesced, or rebuilt. hidden immutable engine cut. F6b5 routes that cut through the existing served HTTP/remote-CLI/OpenAPI export surface; it adds no status fields. F7b exposes a graph-redacted HTTP/OpenAPI/remote-CLI view of F6b6's checked read-only - operational-status core. The remaining minimum controls—explicit fold, - persistent quiesce, resume/abort-drain, rebuild execution, general data + operational-status core. F7c exposes graph-wide resume and checked `SEALED` + EnsureIndices/Optimize through the same served transports without exposing a + declaration/table/lane selector. The remaining minimum controls—explicit fold, + persistent quiesce, per-declaration abort-drain, rebuild execution, general data correction, and authority repair—remain inactive. Direct SDK checked status - also remains inactive. + and control also remain inactive. Embedded durable-only status is already active. F6a adds a separate typed, failpoints-only process-local advisory driver snapshot for tests; it does not add fields to that durable projection, and pending triggers are not backlog. @@ -6180,8 +6222,8 @@ remain concurrent with one another. | R0 | production-neutral retained-growth/source audit; current-object census; referenced-cut retry; legal high-entropy near-cap materialize/fold cell; no schema, public caller, or deletion | **Historical bounded-retention no-go 2026-07-20; disposition amended 2026-07-21 (§0.2/§12.4):** RC.1 still exposes neither a complete reserve-first physical envelope/receipt nor a durable cross-open randomized-attempt cap. Those facts prohibit a finite storage promise but do not block selected unbounded retain-all. The formerly red widest cell is now green locally and on the configured-RustFS CI path; current-object observations remain advisory retention evidence, not provider billing/accounting | | B2a | selected unbounded retain-all/no-GC profile on stock Lance | **Private gate implemented 2026-07-21 (§12.5):** no OmniGraph byte/object/file/history quota; zero canonical `_mem_wal` deletion; complete/partial provider residue remains retained, unreferenced, and untouched below its root through retry/reopen; provider failures are loud; local/configured-RustFS history sweeps are advisory. This gate itself activated no schema or product surface; the later private B2-common slice activates v9 | | B2b | candidate managed-reclamation retention profile | Inactive. Requires the Lance-owned durable inspect/plan/execute + receipt, post-success fencing, bounded checkpoint/inventory/accounting, local/RustFS enforced-bound validation, and the profile-specific crash matrix (§4.5.2/§12.6). Passing it alone activates no product surface | -| B2-common | schema v9/config-v3/state-v2, compare-and-chain token/attribution, graph-global token authority, recovery-v12 base+token fold; then explicit enrollment, revision-fenced lifecycle/correction/full status, SDK row/control methods, HTTP, CLI, and OpenAPI | **Private row/fold subset implemented 2026-07-22 (§11/§12.6):** canonical digests, hidden attribution, stale-authority revalidation after shared admission, same-generation chains, exact two-participant recovery/publication, durable fold attribution, retain-all, and genuine v8↔v9 refusal/rebuild are green. Explicit production enrollment and general lifecycle mutation remain inactive. Later EXP slices activate graph-native served row ingress (F7a) and the graph-redacted checked HTTP/OpenAPI/remote-CLI status projection (F7b); direct SDK status and lifecycle/maintenance parity remain inactive. The Cedar vocabulary, embedded manifest-only status, and narrow stopped/offline F3f DataBlock correction also shipped in EXP slices. `GraphHistoryBudget` belongs only to a future bounded/managed profile | -| EXP | experimental cluster-only activation of the §4.7 profile: offline capability-bound enablement, lazy enrollment, caller-supplied vectors, terminal per-key dead letter plus correction, irreversible authority retirement for fresh-root rebuild, SEALED maintenance/rebind, starvation-free serial folding, graph-native served ingress, and graph-redacted served status | **Selected 2026-07-27 and amended through F7b (§4.7); F3a–F3f, hidden F4, F5a/F5b0/F5b, F6a–F6b8 evidence subsets, F7a graph ingress, and F7b graph status are implemented.** Current v19/token-schema-v3/recovery-v21 publishes deterministic mixed/all-diverted folds, current `DEAD_LETTERED` authority, exact retry/ordinary successor, stopped/offline inspection, and three-disposition retirement. F6b1 freezes an exact-terminal move-only cut; F6b5 connects it to existing served HTTP/remote-client/CLI/OpenAPI export with incremental exact-version scans using approximate Lance targets, strict 64-KiB chunks, complete queue-envelope reservation, pre-header typed refusal, backpressure, and disconnect-safe body-plus-producer ownership. F6b6 adds the checked operational cut with explicit checked `DISABLING` cluster-apply status authority. F6b7 adds a paired failpoints-only exact-selected token-index decision instrument without recovery or production maintenance. Within the hard status envelope status reports every sidecar as rebuild-blocking, while an over-bound discovery refuses the whole cut; it makes only exact sidecar-owned base-HEAD movement physically unavailable and reports cold-replay/flushed-LWW accounting plus exact oldest-uncovered age as unavailable; the public manifest-only status is unchanged. F7a exposes one graph-only mixed node/edge NDJSON route and remote command over the existing checked runtime, hidden lazy enrollment, and resident fold driver. A strong graph-authority ETag gates body ownership; results are graph-logical and redact all table/lane/binding evidence. F7b exposes the logical checked status cut with `read` authorization and `no-store`, omitting physical and opaque control identities. Public lane enrollment, lifecycle/rebind control, served maintenance, direct SDK parity, and unreachable `AuthorityBlock` repair remain inactive. F6b7's bounded NO-GO applies only to the uncompacted profile-cycle fixture and schedules no standalone production token-index reconciler; remeasurement begins beyond 260 uncovered fragments, after a Lance/index-grammar change, or before considering graph-manifest-compacted or checked-Optimize-coupled maintenance. | +| B2-common | schema v9/config-v3/state-v2, compare-and-chain token/attribution, graph-global token authority, recovery-v12 base+token fold; then explicit enrollment, revision-fenced lifecycle/correction/full status, SDK row/control methods, HTTP, CLI, and OpenAPI | **Private row/fold subset implemented 2026-07-22 (§11/§12.6):** canonical digests, hidden attribution, stale-authority revalidation after shared admission, same-generation chains, exact two-participant recovery/publication, durable fold attribution, retain-all, and genuine v8↔v9 refusal/rebuild are green. Explicit production enrollment and per-declaration/general lifecycle mutation remain inactive. Later EXP slices activate graph-native served row ingress (F7a), the graph-redacted checked HTTP/OpenAPI/remote-CLI status projection (F7b), and selector-free graph-wide resume plus checked `SEALED` EnsureIndices/Optimize (F7c); direct SDK status/control, per-declaration abort, and public rebind remain inactive. The Cedar vocabulary, embedded manifest-only status, and narrow stopped/offline F3f DataBlock correction also shipped in EXP slices. `GraphHistoryBudget` belongs only to a future bounded/managed profile | +| EXP | experimental cluster-only activation of the §4.7 profile: offline capability-bound enablement, lazy enrollment, caller-supplied vectors, terminal per-key dead letter plus correction, irreversible authority retirement for fresh-root rebuild, SEALED maintenance/rebind, starvation-free serial folding, graph-native served ingress, graph-redacted served status, and graph-wide resume/SEALED maintenance | **Selected 2026-07-27 and amended through F7c (§4.7); F3a–F3f, hidden F4, F5a/F5b0/F5b, F6a–F6b8 evidence subsets, F7a graph ingress, F7b graph status, and F7c graph controls are implemented.** Current v19/token-schema-v3/recovery-v21 publishes deterministic mixed/all-diverted folds, current `DEAD_LETTERED` authority, exact retry/ordinary successor, stopped/offline inspection, and three-disposition retirement. F6b1 freezes an exact-terminal move-only cut; F6b5 connects it to existing served HTTP/remote-client/CLI/OpenAPI export with incremental exact-version scans using approximate Lance targets, strict 64-KiB chunks, complete queue-envelope reservation, pre-header typed refusal, backpressure, and disconnect-safe body-plus-producer ownership. F6b6 adds the checked operational cut with explicit checked `DISABLING` cluster-apply status authority. F6b7 adds a paired failpoints-only exact-selected token-index decision instrument without recovery or production maintenance. Within the hard status envelope status reports every sidecar as rebuild-blocking, while an over-bound discovery refuses the whole cut; it makes only exact sidecar-owned base-HEAD movement physically unavailable and reports cold-replay/flushed-LWW accounting plus exact oldest-uncovered age as unavailable; the public manifest-only status is unchanged. F7a exposes one graph-only mixed node/edge NDJSON route and remote command over the existing checked runtime, hidden lazy enrollment, and resident fold driver. A strong graph-authority ETag gates body ownership; results are graph-logical and redact all table/lane/binding evidence. F7b exposes the logical checked status cut with `read` authorization and `no-store`, omitting physical and opaque control identities. F7c exposes selector-free graph-wide resume and graph-wide checked `SEALED` EnsureIndices/Optimize over HTTP/OpenAPI and the remote CLI, reusing recovery-v15/v16/v17 without a new coordinator or format strand. Public lane enrollment, per-declaration/general lifecycle/abort, public rebind, direct SDK parity, and unreachable `AuthorityBlock` repair remain inactive. F6b7's bounded NO-GO applies only to the uncompacted profile-cycle fixture and schedules no standalone production token-index reconciler; remeasurement begins beyond 260 uncovered fragments, after a Lance/index-grammar change, or before considering graph-manifest-compacted or checked-Optimize-coupled maintenance. | | C | restart-stable reject-row identity, atomic dead letter, richer status, and evidence-backed configurable bounds | reject crash matrix; reject-retention proof; backpressure and RSS/latency evidence. The §4.7 profile pulls a bounded object-form dead-letter subset forward using the §4.1 token as reject identity | | D | automatic operation drain, broader schema/branch/upgrade integration, and orchestrated rematerialization rebind beyond P7's explicit bridge | two-coordinator race, old/new physical-binding crash matrix, and format-transition suite | | E | fresh cuts and maintained-index reads; cross-process `Fresh` ships only if the substrate generation-retention guard exists (§9), otherwise same-process only | cut consistency; merged-generation exclusion | diff --git a/docs/rfcs/0028-stable-schema-identity.md b/docs/rfcs/0028-stable-schema-identity.md index 363e97c0..1dcc3d10 100644 --- a/docs/rfcs/0028-stable-schema-identity.md +++ b/docs/rfcs/0028-stable-schema-identity.md @@ -486,9 +486,11 @@ V16/recovery-v18 adds the separate private physical-rebind owner for an exact authority-retirement and receipt-bearing export exit. Current v18/recovery-v20 adds exact stopped/offline `DataBlock` correction through the cluster-only `stream block show|correct` commands. Graph-native served ingress -is active; explicit enrollment, ordinary lifecycle control, authority repair, -rebind, and public maintenance surfaces remain inactive. Retirement and -DataBlock control remain the two narrow cluster-control CLI exceptions. +is active; explicit enrollment, per-declaration lifecycle control, authority +repair, and rebind remain inactive. Graph-wide checked resume and sealed +EnsureIndices/Optimize are the only served lifecycle/maintenance controls; +they expose no type/table selector and reuse recovery-v15/v16/v17 unchanged. +Retirement and DataBlock control remain narrow cluster-control CLI exceptions. None of the later formats reinterprets or backfills v5 in place. A v5 graph was never served with a diff --git a/docs/user/cli/reference.md b/docs/user/cli/reference.md index 8ac92edd..d6367054 100644 --- a/docs/user/cli/reference.md +++ b/docs/user/cli/reference.md @@ -13,6 +13,8 @@ Top-level command families and subcommands. Graph-targeting commands accept a po | `ingest` | deprecated alias of `load --from ` (defaults: `--from main --mode merge`); prints a one-line warning to stderr | | `stream ingest` | experimental graph-native firehose; served-only. `stream ingest [--data ] [--graph-token ]` streams mixed node/edge NDJSON to `/graphs/{graph_id}/stream/ingest`, performs at most one missing-token preflight before opening the body, and emits ordered NDJSON acknowledgement lines. `--data` defaults to `-` (stdin). It never targets a table or direct `--store` | | `stream status` | checked operational status for one served streaming graph. `stream status [--json]` reports graph/profile readiness, lifecycle revisions for logical node/edge declarations whose streaming state has initialized, pending-work availability, terminal-authority counts, driver/recovery health, and rebuild blockers. A declaration absent from `enrolled_declarations` may still exist in the graph schema. It never exposes or accepts a table, dataset, lane, binding, shard, or Lance identifier | +| `stream resume` | graph-wide checked resume for one served graph. It reopens every currently `SEALED` enrolled declaration, skips declarations already `OPEN`, and refuses before effects when a drain or strict block still needs attention. There is deliberately no type, table, or lane selector | +| `stream maintenance ensure-indices \| optimize` | graph-wide checked maintenance. Any enrolled declaration the operation would physically change must be `SEALED`; an unaffected `OPEN` declaration does not block derived maintenance. Both commands are served-only, use the existing coordinated graph publication/recovery paths, and return aggregate results only; physical dataset and fragment identities remain private | | `query ` (alias: `read`) | run a read query. **Catalog lane** (default): `` is a stored query invoked **by name** from the served catalog (served-only — address with `--server`/`--profile`; the verb asserts the query is a read). **Ad-hoc lane**: with `--query ` or `-e`/`--query-string `, runs that source (the positional `` then selects which query in it). No positional graph URI — address via `--store`/`--server`/`--profile`. `read` is the deprecated previous name (one-line stderr warning) | | `mutate ` (alias: `change`) | run a mutation query; same catalog (by-name, served-only, verb asserts mutation) / ad-hoc (`--query`/`-e`) lanes as `query`. `change` is the deprecated previous name (one-line stderr warning) | | `alias [args]` | invoke an operator alias — a read-only personal binding (under `aliases:` in `~/.omnigraph/config.yaml`) to a stored query on a named server (replaces the removed `--alias` flag; stored mutations are rejected before execution) | @@ -38,7 +40,7 @@ Top-level command families and subcommands. Graph-targeting commands accept a po Every command declares the **capability** it needs — what it requires to reach a graph — which determines the addressing flags that apply: - **`any`** — `query`, `mutate`, `load`, `ingest`, `branch *`, `snapshot`, `export`, `commit *`, `schema show`, `schema apply`. Run against a graph **served (via a server) or embedded (direct against a store)**: accept a positional `file://`/`s3://` URI, `--server ` (+ `--graph ` for multi-graph servers), `--store `, or `--profile `. A remote server is addressed with `--server` — a positional `http(s)://` URI does **not** dispatch to one. -- **`served`** — `graphs list`, `stream ingest`, and `stream status`. All require a server. `graphs list` addresses the graph *registry* (the bare server URL), while the `stream` commands require one selected graph through `--server` / `--profile` plus `--graph` or a configured default. Direct `--store` is rejected for both stream commands; client-supplied `--as` is additionally rejected because the server resolves the authenticated actor. +- **`served`** — `graphs list` and the `stream ingest|status|resume|maintenance` family. All require a server. `graphs list` addresses the graph *registry* (the bare server URL), while every `stream` command requires one selected graph through `--server` / `--profile` plus `--graph` or a configured default. Direct `--store` and client-supplied `--as` are rejected; the server resolves the authenticated actor for writes. - **`direct`** — `init`, `optimize`, `repair`, `cleanup`, `schema plan`, `lint`. Need **direct storage access** (`file://` / `s3://`), never through a server. They accept a positional `URI`, but **not** `--server`, and a remote (`http(s)://`) URI is rejected. `optimize` / `repair` / `cleanup` additionally accept **`--cluster --graph `** (`--cluster` is a cluster directory or storage-root URI, named via `clusters:` in `~/.omnigraph/config.yaml` or a literal root), which resolves the graph's storage URI from the served cluster state (so you needn't know the `/graphs/.omni` layout). `--graph` is the one graph selector across all scopes — on these three verbs it picks the cluster graph; on the other `direct` verbs it does not apply. `--as` does not apply to any `direct` verb — maintenance records no actor. - **`control`** — `cluster *` via `--config `; `policy *` and `queries *` via `--cluster ` or a cluster profile. - **`local`** — `alias`, `embed`, `login`, `logout`, `profile`, `version`. Address no explicit graph scope. @@ -194,11 +196,16 @@ omnigraph cluster status --config company-brain --json omnigraph cluster refresh --config company-brain --json omnigraph cluster import --config company-brain --json omnigraph cluster force-unlock --config company-brain --json +# After re-enabling a graph whose declarations are sealed: +omnigraph stream resume --server --graph --json +# Checked graph-wide maintenance (affected enrolled declarations must be sealed): +omnigraph stream maintenance ensure-indices --server --graph --json +omnigraph stream maintenance optimize --server --graph --json # Inspect and correct one exact strict drain block while every writer is stopped: -omnigraph --graph --as cluster stream block show node:Person \ +omnigraph --graph --as cluster stream block show \ --config company-brain --block-token \ --confirm-stream-offline --json -omnigraph --graph --as cluster stream block correct node:Person \ +omnigraph --graph --as cluster stream block correct \ --config company-brain --block-token --correction-id \ --expected-lifecycle-revision --plan correction.json \ --confirm-stream-offline --json @@ -236,9 +243,12 @@ apply --confirm-stream-offline` is additionally required when a streaming profile changes. It attests that every writer-capable process for the affected graph is stopped; profile changes also require the state lock and an authenticated `--as` actor. The flag is not a distributed lease. Enabling the -experimental profile disables embedded/direct Mutation/Load/delete, does not -yet add a public firehose endpoint, and should be followed by a server restart -so served writes carry the checked runtime authority. Branch merge remains +experimental profile disables embedded/direct Mutation/Load/delete and should +be followed by a server restart so the graph-native firehose and controls carry +the checked runtime authority. The first ingest lazily initializes internal +declarations. Re-enabling does not implicitly reopen a previously sealed +declaration; run the selector-free, graph-wide `stream resume` after restart. +Branch merge remains unavailable while that profile is `ENABLED` or `DISABLING`, even through the served runtime. Offline disable publishes `DISABLING`, derives one finite manifest lane cut, and serially drains `OPEN`, goal-`SEALED`, and adopted @@ -253,7 +263,9 @@ removes a lock only when the supplied id exactly matches the lock file. cluster recovery, and `--confirm-stream-offline`. The flag is an operator attestation, not a distributed lease. `show` revalidates the exact block and returns at most one bounded page; pass its opaque `--cursor` to fetch the next -page. `correct` accepts a strict JSON file with no unknown fields: +page. The block token resolves the affected internal declaration; neither +command accepts a type, table, or dataset selector. `correct` accepts a strict +JSON file with no unknown fields: ```json { @@ -284,7 +296,8 @@ manifest-selected current-token version and return one bounded page; pass the opaque `--cursor` for the next page. `list` returns current `DEAD_LETTERED` sequencing evidence. `export` additionally verifies the recovery-owned object descriptor and returns canonical payload entries. It never lists the object -prefix or treats payload export as replay/import. +prefix or treats payload export as replay/import. Entries identify their +accepted-schema node/edge declaration, not a physical table or dataset. `cluster stream retire-for-rebuild` is separate from normal apply. Both `plan` and `confirm` require a declared/applied graph, `state.lock: true`, an diff --git a/docs/user/clusters/config.md b/docs/user/clusters/config.md index da4b1294..b4a1409e 100644 --- a/docs/user/clusters/config.md +++ b/docs/user/clusters/config.md @@ -169,9 +169,10 @@ arming, or Lance effects. Existing served Mutation/Load operations continue only through the one cluster-booted server carrying the exact checked runtime authority. The graph-native served firehose is available through `POST /graphs/{graph_id}/stream/ingest` and remote `stream ingest`, but only an -absent lane (prepared lazily) or an existing `OPEN` lane can admit rows in F7a. -An existing `SEALED` lane after disable/re-enable needs the later public resume -surface; repeated ingest requests do not reopen it. +absent declaration (prepared lazily) or an existing `OPEN` declaration can +admit rows. After disable/re-enable, restart the checked server and run the +selector-free graph-wide `omnigraph stream resume` to reopen every `SEALED` +declaration. Repeated ingest requests never reopen one implicitly. Branch merge is stricter: it is refused while the profile is `ENABLED` or `DISABLING`, even through that checked served runtime, because this release has no token-aware merge transition. diff --git a/docs/user/clusters/index.md b/docs/user/clusters/index.md index 1e28ad99..9ab29489 100644 --- a/docs/user/clusters/index.md +++ b/docs/user/clusters/index.md @@ -200,15 +200,29 @@ Mutation/Load/delete fail before input reads or durable effects. Existing served mutations work only through the restarted cluster-booted server's checked runtime authority. Graph-native producers use the served [`/stream/ingest` firehose](../operations/server.md#streaming), which -keeps physical datasets and lanes private. F7a admits absent or `OPEN` lanes; -it does not resume a `SEALED` lane after disable/re-enable. Branch merge remains refused while the profile is `ENABLED` or +keeps physical datasets and lanes private. Ingest admits absent or `OPEN` +internal declarations. After disable/re-enable, run the graph-wide served +`omnigraph stream resume`; it opens every `SEALED` declaration and exposes no +per-type, table, or lane selector. Branch merge remains refused while the profile is `ENABLED` or `DISABLING`, including through the checked server runtime. A later explicit `streaming: false` offline apply publishes `DISABLING`, derives one finite manifest lane cut, and serially drains `OPEN`, goal-`SEALED`, and adopted `OPEN_AFTER_FOLD` lanes. A selected `DataBlock` leaves the apply pending until -stopped/offline correction and a retry. Only the no-lane case restores the direct physical lane. -Already-`SEALED` enrollments remain fenced, so use the strict export/init/load -rebuild to return an enrolled graph to that non-streaming lane. +stopped/offline correction and a retry. Only the no-lane case restores the +direct physical lane. A disabled enrolled graph remains a checked +served/export state; resume is available only after the profile is enabled and +the cluster-booted server is restarted. + +The sealed window is also the supported maintenance window. After applying +`streaming: true` and restarting—but before `stream resume`—run +`stream maintenance ensure-indices` and/or `stream maintenance optimize`. +These controls operate on the whole graph through the existing coordinated +manifest/recovery paths. They do not accept a declaration selector and return +only aggregate results. Resume is convergent rather than a new all-dataset +transaction: it preflights the complete graph, then reopens internal +declarations in deterministic order; if an unexpected race interrupts the +sequence, retrying the same graph-level command skips declarations already +`OPEN` and continues the remainder. ### Strict drain blocks: inspect → correct → retry @@ -217,19 +231,21 @@ and inspect the exact blocked cut through the cluster control plane: ```bash omnigraph --graph knowledge --as andrew \ - cluster stream block show node:Person \ + cluster stream block show \ --config company-brain --block-token \ --confirm-stream-offline --json omnigraph --graph knowledge --as andrew \ - cluster stream block correct node:Person \ + cluster stream block correct \ --config company-brain --block-token \ --correction-id --expected-lifecycle-revision \ --plan correction.json --confirm-stream-offline --json ``` `show` reconstructs validator evidence from the retained immutable WAL -generation and returns a bounded page; follow `next_cursor` until it is absent. +generation and returns a bounded page; the opaque block token resolves the +affected internal declaration, so the user never supplies a type/table/lane +selector. Follow `next_cursor` until it is absent. Build an ordered plan that chooses `REPLACE` or `WITHDRAW` for the entries it changes; unmentioned keys retain their blocked winner, and the resulting complete overlay must clear every violation. `correct` revalidates the block, @@ -260,13 +276,16 @@ omnigraph --graph knowledge --as andrew \ Follow `next_cursor` with `--cursor` until it is absent. Both commands pin the manifest-selected token version; export verifies the recovery-owned object descriptor and does not prefix-list storage. Payload export is an inspection -artifact, not replay or import. A fresh ordinary stream occurrence can restore +artifact, not replay or import. Each entry names its logical node/edge +declaration while keeping the physical table and dataset private. A fresh ordinary stream occurrence can restore `PRESENT` by naming the terminal token as predecessor. The graph-native `stream ingest` command and `POST /graphs/{graph_id}/stream/ingest` route can submit that occurrence while the enabled lane is absent or `OPEN`; payload export itself does not replay it -automatically. Once the lane is terminally disabled/`SEALED`, retirement and -rebuild remain the public exit until resume control ships. +automatically. If the declaration is `SEALED`, re-enable/restart the served +graph and run graph-wide `stream resume` before submitting the successor. +Retirement and rebuild remain the irreversible exit when terminal sequencing +authority must be discarded for export. ### Terminal authority retirement: plan → confirm → rebuild diff --git a/docs/user/operations/errors.md b/docs/user/operations/errors.md index 8969f34d..f3fef658 100644 --- a/docs/user/operations/errors.md +++ b/docs/user/operations/errors.md @@ -46,9 +46,11 @@ ordinary export found current `WITHDRAWN` or `DEAD_LETTERED` sequencing authority that a row-only artifact cannot preserve. Graph-native firehose ingest can install a fresh ordinary `PRESENT` successor only while the - enabled declaration lane is absent or `OPEN`. A terminally disabled/ - `SEALED` lane has no public resume in F7a; its operator exit is the stopped/offline - `cluster stream retire-for-rebuild` handshake. + enabled declaration is absent or `OPEN`. For a disabled `SEALED` + declaration, re-enable streaming, restart the checked server, run the + selector-free graph-wide `omnigraph stream resume`, and then submit the + successor. If the authority must instead be discarded, use the stopped/ + offline `cluster stream retire-for-rebuild` handshake. The served export route returns this as HTTP **409** before `200`. - Graph-ingest HTTP preconditions are transport errors rather than new durable engine states. Missing `If-Match` returns **428** plus the current opaque diff --git a/docs/user/operations/maintenance.md b/docs/user/operations/maintenance.md index 97cef971..251372dd 100644 --- a/docs/user/operations/maintenance.md +++ b/docs/user/operations/maintenance.md @@ -12,7 +12,7 @@ - **Create declared-but-missing indexes (the index reconciler).** `@index`/`@key` declares intent; `schema apply`, `load`, and `mutate` build no physical indexes inline. They record or publish only their exact logical/data effects and leave all index materialization to `ensure_indices`/`optimize`. `optimize` materializes every buildable declared-but-missing index over the compacted layout — so it is the convergence path for an `@index` added after data exists, or a vector index whose embeddings arrived via a later `embed`. A column still not buildable (no vectors yet) is reported on the table's stat as `pending_indexes` (visible in `--json`), not treated as a failure; the next `optimize` retries. So `optimize` is the single operator-facing index reconciler: it compacts, restores coverage, **and** builds declared-but-missing indexes. - Optimize plans under one schema/main/all-table envelope, writes one identity-bearing v9 recovery envelope with a bounded maintenance payload for the complete productive table set, runs per-table compact→reindex work in bounded parallelism, and publishes the resulting pointers together. A crash after every table effect rolls the batch forward on the next read-write open; a partial effect set is compensated before any pointer becomes graph-visible. Because this record does not prove exact maintenance transaction identity or distributed ownership, destructive recovery is supported only within the documented single-writer-process boundary; it is not a multi-process recovery fence. - **Requires a recovered graph.** `optimize` refuses (errors) when a pending crash-recovery operation is present — operating on an unrecovered graph could publish a partial write that recovery would roll back. Reopen the graph to run recovery, then re-run `optimize`. -- **Uncovered drift is skipped, not interpreted.** If a table's underlying version is ahead of the version recorded in `__manifest` and no crash-recovery record covers that movement, `optimize` reports `skipped: DriftNeedsRepair` with the manifest/head versions and leaves the table untouched. Run `omnigraph repair` to classify and explicitly publish that drift. +- **Uncovered drift is skipped, not interpreted.** If a table's underlying version is ahead of the version recorded in `__manifest` and no crash-recovery record covers that movement, `optimize` reports `skipped: DriftNeedsRepair` with the manifest/head versions and leaves the table untouched. For an ordinary, unenrolled table, run `omnigraph repair` to classify and explicitly publish that drift. Enrolled streaming drift has no supported in-place repair: stop serving, preserve the affected root for investigation, and rebuild a fresh graph from the last verified clean export or backup. - Bounded by `OMNIGRAPH_MAINTENANCE_CONCURRENCY` (default 8). - Returns per-table stats: `table_key, fragments_removed, fragments_added, committed, skipped, manifest_version, lance_head_version, pending_indexes` (the last lists any declared `@index` column the reconciler could not build this run, with the reason — e.g. a vector column with no trainable vectors yet). - **Blob tables use the normal compaction and reindex path.** Lance 8.0.0+ supports blob-v2 compaction, so OmniGraph no longer has a blob-specific skip or capability gate. Fragment reclamation and index-coverage repair therefore apply to blob-bearing tables like every other table. @@ -24,6 +24,7 @@ - Classifies drift by reading the table's transaction history from `manifest_version + 1` through the current head. Only fragment-reservation and rewrite (compaction) operations are verified maintenance. Semantic operations such as append, delete, update, merge, or missing transaction history are not auto-healed. - Publishes repair by advancing `__manifest` to the existing head; it does **not** rewrite data. If the publish succeeds, normal reads and strict writes use the repaired version. If it fails, no new data-side partial state was created. - Requires a clean recovery state. A pending crash-recovery operation still belongs to automatic recovery, not manual repair. +- **Does not repair enrolled streaming drift.** Do not use `repair` to adopt an unexplained head for a streaming declaration: that would bypass its sequencing authority. Stop serving, preserve the affected root for investigation, and rebuild a fresh graph from the last verified clean export or backup. ## `cleanup` — destructive @@ -65,8 +66,10 @@ guessing that its referenced versions are disposable. - **Refuses uncovered main-table drift.** Every manifest-visible main version must open and equal Lance HEAD during the graph-wide preflight. If an external - or interrupted operation advanced HEAD without a recovery sidecar, run - `omnigraph repair` before cleanup; version GC never guesses around that drift. + or interrupted operation advanced HEAD without a recovery sidecar, classify + ordinary unenrolled drift with `omnigraph repair` before cleanup. For enrolled + streaming drift, follow the preserve-and-rebuild posture above; version GC + never guesses around either kind of drift. - **Orphaned-branch reconciliation:** before the version GC, cleanup reclaims any per-table Lance branch absent from the manifest branch list. These orphans arise when a `branch_delete` flips the manifest authority but a downstream best-effort reclaim does not complete (see [branches-commits.md](../branching/index.md)). The reconciler is idempotent (it no-ops once nothing is orphaned), runs regardless of the `keep_versions` / `older_than` values (those gate version GC only), and never reclaims `main` or system-branch forks. Reclaimed forks are logged. Graph lineage has no separate branch dataset: it lives in `__manifest`. ## Tombstones diff --git a/docs/user/operations/policy.md b/docs/user/operations/policy.md index 1df635cc..a7e52666 100644 --- a/docs/user/operations/policy.md +++ b/docs/user/operations/policy.md @@ -15,7 +15,7 @@ Per-graph actions (bind to `Omnigraph::Graph::""`): 7. `branch_merge` 8. `admin` — reserved for policy-management surfaces (hot reload, audit log, approvals). No call site today. 9. `stream_ingest` — **experimental (RFC-026)**; gates the graph-native firehose at `POST /graphs/{graph_id}/stream/ingest`. The graph is the authorization resource: logical node/edge declarations are row data, while the private Lance datasets and MemWAL lanes are never policy resources. The profile is main-only, so a rule that sets `branch_scope` or `target_branch_scope` is rejected at validation. Deliberately separate from `change`: a stream append acknowledges *durability* without graph visibility, so an operator can grant high-rate ingestion without granting direct-lane writes, or the reverse. -10. `stream_manage` — **experimental (RFC-026)**; gates streaming *lifecycle management*: enabling or disabling the graph's streaming flag today, plus fold / quiesce / resume / abort-drain as they ship. Graph-scoped for the same main-only reason. Split from `stream_ingest` because the blast radii differ in kind — ingestion adds rows, while management can seal a lane, drain acknowledged data, or reopen it at a new epoch. Read-only stream *status* is authorized like other graph operational metadata rather than by this action, so an operator can always see whether a lane is stuck even without the rights to act on it. +10. `stream_manage` — **experimental (RFC-026)**; gates streaming lifecycle management: cluster-owned enable/disable and stopped controls, plus the served selector-free graph-wide resume and checked graph-wide EnsureIndices/Optimize controls. Graph-scoped for the same main-only reason. Split from `stream_ingest` because the blast radii differ in kind — ingestion adds rows, while management can quiesce acknowledged data, reopen sealed declarations at a new epoch, or move derived storage state. Read-only stream *status* is authorized like other graph operational metadata rather than by this action, so an operator can always see whether the graph is stuck even without the rights to act on it. 11. `invoke_query` — gates invoking a server-side stored query (the `queries:` registry). Graph-scoped (like `admin`) — per-branch access is enforced by the inner `read` / `change` gate, so a rule that sets `branch_scope` on `invoke_query` is rejected. Coarse in this release: an `invoke_query` allow rule permits any stored query on the graph; a future, additive refinement adds an optional per-query-name scope without changing rules written against the coarse action. Enforced at `POST /queries/{name}` (see [server](server.md)). A stored *mutation* is double-gated: `invoke_query` to reach the tool, plus `change` for the write itself (the engine `_as` writers still enforce per the query body). Server-scoped action (v0.6.0+; binds to `Omnigraph::Server::"root"`): diff --git a/docs/user/operations/server.md b/docs/user/operations/server.md index 851e3673..c82d54cc 100644 --- a/docs/user/operations/server.md +++ b/docs/user/operations/server.md @@ -219,9 +219,48 @@ graph blocker is resolved. Disconnect stops future body polling while already-invoked work remains owned until its durability result is classified. -F7a admits an absent declaration lane (prepared lazily) or an existing `OPEN` -lane. It does not expose resume: a `SEALED` lane after disable/re-enable returns -`stream_authority_changed` until a later graph-level resume surface exists. +Ingest admits an absent internal declaration (prepared lazily) or an existing +`OPEN` one. A `SEALED` declaration returns `stream_authority_changed`; it is +never resumed as a side effect of ingest. + +`POST /graphs/{graph_id}/stream/resume` is the explicit graph-wide resume +control. It accepts no body and no declaration selector. Under exact checked +`ENABLED` runtime authority it preflights every enrolled declaration, refuses +before effects if any declaration is still draining or strictly blocked, +skips declarations already `OPEN`, and reopens the `SEALED` remainder in a +deterministic internal order. A retry after an interrupted sequence is +convergent: declarations already opened are skipped. Physical table, lane, +claim, and recovery identities never appear in the response. + +Two bodyless graph-wide maintenance controls are available. Any enrolled +declaration that the operation would physically change must be `SEALED`; an +unaffected `OPEN` declaration does not block derived maintenance: + +- `POST /graphs/{graph_id}/stream/maintenance/ensure-indices` +- `POST /graphs/{graph_id}/stream/maintenance/optimize` + +They reuse OmniGraph's existing graph coordinator and Lance maintenance +adapters; they do not introduce a second WAL, job queue, or persisted +coordinator. Productive work publishes through the existing one-graph +recovery/manifest envelope, while the response contains aggregate change, +pending-index, and repair-needed status only. There is no per-type, per-table, +or per-dataset endpoint. + +The control plane does not turn graph scope into one physical dataset. Resume +captures the manifest work list once, then performs the necessary +recovery-covered transitions against the separate Lance datasets in a bounded, +deterministic sequence because the experimental profile has one resident WAL +writer slot. Ensure-indices and Optimize already plan the graph once; Optimize +runs productive physical tasks with bounded concurrency and publishes their +visibility together. Clients therefore make one graph request and do not pay a +public request round trip per declaration, while the coordinator still honors +the real multi-dataset atomicity boundary. + +All three controls require graph-scoped `stream_manage`. The actor comes only +from the server's bearer-token resolution; clients cannot supply one. Runtime, +recovery, and storage errors are mapped to graph-safe responses that omit +table keys, dataset locations, manifest/Lance coordinates, and internal +operation identifiers. `GET /graphs/{graph_id}/stream/status` returns one checked, read-only operational cut for a cluster-served streaming graph. The response is diff --git a/docs/user/operations/upgrade.md b/docs/user/operations/upgrade.md index dc2bb4dd..999d16f3 100644 --- a/docs/user/operations/upgrade.md +++ b/docs/user/operations/upgrade.md @@ -206,9 +206,11 @@ inspection artifact, not a replay or import protocol. A corrected value is a fresh ordinary stream admission naming the current terminal token as its predecessor; while it remains current, exact retry returns the same terminal result. The graph-native HTTP/remote-CLI firehose can submit that successor only -while the enabled declaration lane is absent or `OPEN`. A terminally disabled/ -`SEALED` lane cannot be reopened by F7a ingress; until public resume ships, -retirement/rebuild remains the public exit there. +while the enabled declaration is absent or `OPEN`. For a disabled `SEALED` +declaration, re-enable streaming, restart the checked server, run the +selector-free graph-wide `omnigraph stream resume`, and then submit the +successor. Ingest never reopens it implicitly. Retirement/rebuild remains the +exit when that sequencing authority must intentionally be discarded. If terminal authority must intentionally be discarded for a fresh-root rebuild, use the same v19 binary's irreversible diff --git a/openapi.json b/openapi.json index 396107df..9dea3fb3 100644 --- a/openapi.json +++ b/openapi.json @@ -1928,6 +1928,328 @@ ] } }, + "/graphs/{graph_id}/stream/maintenance/ensure-indices": { + "post": { + "tags": [ + "streaming" + ], + "summary": "Refresh graph index state; any enrolled declaration changed must be sealed.", + "operationId": "cluster_stream_ensure_indices", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Indexes were checked across the graph; affected enrolled declarations were sealed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StreamEnsureIndicesOutput" + } + } + } + }, + "400": { + "description": "The bodyless graph control received a request body", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "The actor is not authorized for graph stream management", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "The graph is not in the sealed maintenance posture", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "413": { + "description": "The bounded graph management operation exceeded a hard limit", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "500": { + "description": "Graph index maintenance failed without exposing physical diagnostics", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "Graph recovery must complete before maintenance can proceed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, + "/graphs/{graph_id}/stream/maintenance/optimize": { + "post": { + "tags": [ + "streaming" + ], + "summary": "Optimize the graph; any enrolled declaration changed must be sealed.", + "operationId": "cluster_stream_optimize", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The graph was considered for optimization; affected enrolled declarations were sealed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StreamOptimizeOutput" + } + } + } + }, + "400": { + "description": "The bodyless graph control received a request body", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "The actor is not authorized for graph stream management", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "The graph is not in the sealed maintenance posture", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "413": { + "description": "The bounded graph management operation exceeded a hard limit", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "500": { + "description": "Graph stream optimization failed without exposing physical diagnostics", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "Graph recovery must complete before maintenance can proceed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, + "/graphs/{graph_id}/stream/resume": { + "post": { + "tags": [ + "streaming" + ], + "summary": "Reopen every sealed streaming declaration in the graph.", + "description": "The operation is deliberately bodyless and graph-wide: callers cannot\nselect a logical type, table, lane, dataset, or physical maintenance\ntarget. The server supplies the bearer-resolved actor and the engine\nperforms the authoritative graph-scoped `stream_manage` check.", + "operationId": "cluster_stream_resume", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "All sealed streaming declarations were reopened through graph authority", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StreamResumeOutput" + } + } + } + }, + "400": { + "description": "The bodyless graph control received a request body", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "The actor is not authorized for graph stream management", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "The graph streaming profile or lifecycle is not ready to resume", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "413": { + "description": "The bounded graph management operation exceeded a hard limit", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "500": { + "description": "Graph stream resume failed without exposing physical diagnostics", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "Graph recovery must complete before resume can proceed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, "/graphs/{graph_id}/stream/status": { "get": { "tags": [ @@ -3331,6 +3653,24 @@ } } }, + "StreamEnsureIndicesOutput": { + "type": "object", + "description": "Aggregate result of graph-wide checked index refresh. Any enrolled\ndeclaration changed by the operation is required to be sealed.", + "required": [ + "changed", + "pending_index_count" + ], + "properties": { + "changed": { + "type": "boolean" + }, + "pending_index_count": { + "type": "integer", + "format": "int64", + "minimum": 0 + } + } + }, "StreamIngestChallenge": { "type": "object", "description": "Effect-free precondition challenge for graph-native streaming ingest.\n\nThe same value is returned as a strong `ETag` response header. Clients\nretry the request with that tag in `If-Match`; this convenience copy keeps\nthe graph authority token distinct from the per-row sequencing\n[`StreamIngestLineOutput::stream_token`].", @@ -3540,6 +3880,28 @@ "sealed" ] }, + "StreamOptimizeOutput": { + "type": "object", + "description": "Aggregate result of graph-wide checked stream optimization. Any enrolled\ndeclaration changed by the operation is required to be sealed. Physical\nfragment and dataset details stay inside the engine.", + "required": [ + "changed", + "pending_index_count", + "requires_repair" + ], + "properties": { + "changed": { + "type": "boolean" + }, + "pending_index_count": { + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "requires_repair": { + "type": "boolean" + } + } + }, "StreamPendingStatusOutput": { "oneOf": [ { @@ -3781,6 +4143,38 @@ } } }, + "StreamResumeOutput": { + "type": "object", + "description": "Aggregate result of reopening every sealed streaming declaration in a\ngraph. Declaration, table, lane, dataset, and recovery identities are\ndeliberately absent from this graph-level control-plane shape.", + "required": [ + "profile_revision", + "enrolled_declarations", + "resumed_declarations", + "already_open_declarations" + ], + "properties": { + "already_open_declarations": { + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "enrolled_declarations": { + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "profile_revision": { + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "resumed_declarations": { + "type": "integer", + "format": "int64", + "minimum": 0 + } + } + }, "StreamStatusOutput": { "type": "object", "description": "One coherent, graph-redacted operational status cut.", From 0f71857c152334f9665490e7c50a546c76eca306 Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Wed, 5 Aug 2026 04:26:16 +0300 Subject: [PATCH 2/2] test(server): cover graph stream control paths --- crates/omnigraph-server/tests/openapi.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/omnigraph-server/tests/openapi.rs b/crates/omnigraph-server/tests/openapi.rs index 05539fb3..ac35b8c4 100644 --- a/crates/omnigraph-server/tests/openapi.rs +++ b/crates/omnigraph-server/tests/openapi.rs @@ -172,6 +172,9 @@ const EXPECTED_PATHS: &[&str] = &[ "/graphs/{graph_id}/export", "/graphs/{graph_id}/stream/status", "/graphs/{graph_id}/stream/ingest", + "/graphs/{graph_id}/stream/resume", + "/graphs/{graph_id}/stream/maintenance/optimize", + "/graphs/{graph_id}/stream/maintenance/ensure-indices", "/graphs/{graph_id}/change", "/graphs/{graph_id}/mutate", "/graphs/{graph_id}/queries",